// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
/*
 * Copyright (c) 2019 Amlogic, Inc. All rights reserved.
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/clk.h>
#include <linux/io.h>
#include <linux/hw_random.h>
#include <linux/of_device.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/of.h>
#include <linux/device.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/scatterlist.h>
#include <linux/dma-mapping.h>
#include <linux/delay.h>
#include <linux/crypto.h>
#include <crypto/internal/des.h>
#include <crypto/scatterwalk.h>
#include <crypto/algapi.h>
#include <crypto/des.h>
#include <crypto/hash.h>
#include <crypto/internal/hash.h>
//#include <linux/amlogic/iomap.h>
#include <crypto/skcipher.h>
#include "aml-crypto-dma.h"
#include <crypto/internal/skcipher.h>

/* TDES flags */
#define TDES_FLAGS_MODE_MASK	0x03
#define TDES_FLAGS_ENCRYPT	BIT(0)
#define TDES_FLAGS_CBC		BIT(1)

#define TDES_FLAGS_INIT         BIT(8)
#define TDES_FLAGS_DMA          BIT(9)
#define TDES_FLAGS_FAST         BIT(10)
#define TDES_FLAGS_BUSY         BIT(11)
#define TDES_FLAGS_ERROR        BIT(12)

#define AML_TDES_QUEUE_LENGTH	50

#define DEFAULT_AUTOSUSPEND_DELAY	1000

struct aml_tdes_dev;

struct aml_tdes_ctx {
	struct aml_tdes_dev *dd;

	u32	keylen;
	u32	key[3 * DES_KEY_SIZE / sizeof(u32)];

	u16	block_size;
	struct crypto_skcipher	*fallback;
	u16 same_key;

	int kte;
};

struct aml_tdes_reqctx {
	unsigned long mode;
};

struct aml_tdes_info {
	struct skcipher_alg *algs;
	u32 num_algs;
};

struct aml_tdes_dev {
	struct list_head	list;

	struct aml_tdes_ctx	*ctx;
	struct device		*dev;
	struct device		*parent;

	unsigned long		flags;
	int	err;

	struct aml_dma_dev      *dma;
	u32 thread;
	u32 status;
	u8 link_mode;
	struct crypto_queue	queue;

	struct tasklet_struct	done_task;
	struct tasklet_struct	queue_task;

	struct skcipher_request	*req;
	size_t	total;
	size_t	fast_total;

	struct scatterlist	*in_sg;
	size_t			in_offset;
	struct scatterlist	*out_sg;
	size_t			out_offset;

	size_t	buflen;
	size_t	dma_size;

	void	*buf_in;
	dma_addr_t	dma_addr_in;

	void	*buf_out;
	dma_addr_t	dma_addr_out;

	void	*descriptor;
	dma_addr_t	dma_descript_tab;

	void	*sg_dsc_in;
	dma_addr_t	dma_sg_dsc_in;
	void	*sg_dsc_out;
	dma_addr_t	dma_sg_dsc_out;

	struct aml_tdes_info *info;
};

struct aml_tdes_drv {
	struct list_head	dev_list;
	spinlock_t		lock; /* spinlock for device list */
};

static struct aml_tdes_drv aml_tdes = {
	.dev_list = LIST_HEAD_INIT(aml_tdes.dev_list),
	.lock = __SPIN_LOCK_UNLOCKED(aml_tdes.lock),
};

static int aml_tdes_crypt_dma_stop(struct aml_tdes_dev *dd);
static int set_tdes_kl_key_iv(struct aml_tdes_dev *dd,
			      u32 *key, u32 keylen, u32 *iv)
{
	struct dma_dsc_v2_64 dsc;
	struct device *dev = dd->dev;
	u32 *key_iv = kzalloc(DMA_KEY_IV_BUF_SIZE, GFP_ATOMIC);
	u32 *piv = key_iv;
	u32 len = keylen;
	dma_addr_t dma_addr_key;
	u32 status = 0;
	int err = 0;
	u32 nents = 1;

	if (!key_iv) {
		dev_err(dev, "error allocating key_iv buffer\n");
		return -EINVAL;
	}

	if (iv)
		memcpy(piv, iv, 8);

	len = DMA_KEY_IV_BUF_SIZE; /* full key storage */

	dma_addr_key = dma_map_single(dd->parent, key_iv,
				      DMA_KEY_IV_BUF_SIZE, DMA_TO_DEVICE);

	if (dma_mapping_error(dd->parent, dma_addr_key)) {
		dev_err(dev, "error mapping dma_addr_key\n");
		kfree(key_iv);
		return -EINVAL;
	}

	dsc.src_addr = (u64)(0xffffffffffffff00 | dd->ctx->kte);
	dsc.tgt_addr = 0;
	/* Internal key_iv storage of DMA is 48 bytes (32 for key; 16 for iv)
	 * For some reason, it fails to set key if key length is 8 bytes(ex: DES).
	 * Therefore, we always set length to 32 bytes(full key storage) regardless of
	 * real key length.
	 * PLEASE ask Qian Cheng(cheng.qian@amlogic.com) for details.
	 */
	dsc.length = 32;
	dsc.dsc_cfg.d32 = 0;
	dsc.dsc_cfg.b.mode = MODE_KEY;
	dsc.dsc_cfg.b.owner = 1;
	dsc.dsc_cfg.b.eoc = iv ? 0 : 1;
	aml_dma_dsc_writer(dd->descriptor, 0, &dsc, 0, dd->dma);

	if (iv) {
		dsc.src_addr = (u64)dma_addr_key;
		dsc.tgt_addr = 32;
		dsc.length = 16;
		dsc.dsc_cfg.d32 = 0;
		dsc.dsc_cfg.b.mode = MODE_KEY;
		dsc.dsc_cfg.b.owner = 1;
		dsc.dsc_cfg.b.eoc = 1;
		nents++;
		aml_dma_dsc_writer(dd->descriptor, 1, &dsc, 0, dd->dma);
	}
	status = aml_dma_do_hw_crypto(dd->dma, dd->descriptor, nents, dd->dma_descript_tab,
				      USE_BUSY_POLLING, DMA_FLAG_TDES_IN_USE);
	aml_dma_debug(dd->descriptor, nents, __func__, dd->thread,
			 dd->status, dd->dma);
	if (status & DMA_STATUS_KEY_ERROR) {
		dev_err(dev, "hw crypto failed, status: 0x%x\n", status);
		err = -EINVAL;
	}
	dma_unmap_single(dd->parent, dma_addr_key,
			 DMA_KEY_IV_BUF_SIZE, DMA_TO_DEVICE);

	kfree(key_iv);
	return err;
}

static int set_tdes_key_iv(struct aml_tdes_dev *dd,
			   u32 *key, u32 keylen, u32 *iv)
{
	struct dma_dsc_v2_64 dsc;
	struct device *dev = dd->dev;
	u32 *key_iv = kzalloc(DMA_KEY_IV_BUF_SIZE, GFP_ATOMIC);
	u32 *piv = key_iv + 8;
	u32 len = keylen;
	dma_addr_t dma_addr_key;
	u32 status = 0;
	int err = 0;

	if (!key_iv) {
		dev_err(dev, "error allocating key_iv buffer\n");
		return -EINVAL;
	}

	if (key)
		memcpy(key_iv, key, keylen);

	if (iv)
		memcpy(piv, iv, 8);

	len = DMA_KEY_IV_BUF_SIZE; /* full key storage */

	dma_addr_key = dma_map_single(dd->parent, key_iv,
				      DMA_KEY_IV_BUF_SIZE,
				      DMA_TO_DEVICE);

	if (dma_mapping_error(dd->parent, dma_addr_key)) {
		dev_err(dev, "error mapping dma_addr_key\n");
		kfree(key_iv);
		return -EINVAL;
	}

	dsc.src_addr = (u64)dma_addr_key;
	dsc.tgt_addr = 0;
	dsc.length = len;
	dsc.dsc_cfg.d32 = 0;
	dsc.dsc_cfg.b.mode = MODE_KEY;
	dsc.dsc_cfg.b.owner = 1;
	dsc.dsc_cfg.b.eoc = 1;
	aml_dma_dsc_writer(dd->descriptor, 0, &dsc, 0, dd->dma);

	aml_dma_debug(dd->descriptor, 1, __func__, dd->thread, dd->status, dd->dma);
	status = aml_dma_do_hw_crypto(dd->dma, dd->descriptor, 1, dd->dma_descript_tab,
				      USE_BUSY_POLLING, DMA_FLAG_TDES_IN_USE);
	aml_dma_debug(dd->descriptor, 1, "end tdes keyiv", dd->thread,
			 dd->status, dd->dma);
	if (status & DMA_STATUS_KEY_ERROR) {
		dev_err(dev, "hw crypto failed, status: 0x%x\n", status);
		err = -EINVAL;
	}
	dma_unmap_single(dd->parent, dma_addr_key,
			 DMA_KEY_IV_BUF_SIZE, DMA_TO_DEVICE);

	kfree(key_iv);
	return err;
}

static size_t aml_tdes_sg_copy(struct scatterlist **sg, size_t *offset,
			       void *buf, size_t buflen, size_t total, int out)
{
	size_t count, off = 0;

	while (buflen && total) {
		count = min((*sg)->length - *offset, total);
		count = min(count, buflen);

		if (!count)
			return off;

		scatterwalk_map_and_copy(buf + off, *sg, *offset, count, out);

		off += count;
		buflen -= count;
		*offset += count;
		total -= count;

		if (*offset == (*sg)->length) {
			*sg = sg_next(*sg);
			if (*sg)
				*offset = 0;
			else
				total = 0;
		}
	}

	return off;
}

static size_t aml_tdes_sg_dma(struct aml_tdes_dev *dd, void *descriptor,
			     size_t total)
{
	struct device *dev = dd->dev;
	struct dma_dsc_v2_64 dsc;
	u32 i = 0;
	int err = 0;
	struct scatterlist *in_sg = dd->in_sg;
	struct scatterlist *out_sg = dd->out_sg;
	u32 in_nents = 0, out_nents = 0;
	struct dma_sg_dsc_64 sg_dsc;
	u32 dma_sg_dsc_sz = aml_dma_get_dsc_sz(1, dd->dma);

	in_nents = sg_nents(in_sg);
	out_nents = sg_nents(out_sg);

	if (dd->in_sg != dd->out_sg) {
		err = dma_map_sg(dd->parent, dd->in_sg, in_nents, DMA_TO_DEVICE);
		if (!err) {
			dev_err(dev, "dma_map_sg() error\n");
			return 0;
		}

		err = dma_map_sg(dd->parent, dd->out_sg, out_nents,
				 DMA_FROM_DEVICE);
		if (!err) {
			dev_err(dev, "dma_map_sg() error\n");
			dma_unmap_sg(dd->parent, dd->in_sg, in_nents,
				     DMA_TO_DEVICE);
			return 0;
		}
	} else {
		err = dma_map_sg(dd->parent, dd->in_sg, in_nents,
				 DMA_BIDIRECTIONAL);
		if (!err) {
			dev_err(dev, "dma_map_sg() error\n");
			return 0;
		}
		dma_sync_sg_for_device(dd->parent, dd->in_sg,
				       in_nents, DMA_TO_DEVICE);
	}

	dd->sg_dsc_in = dma_alloc_coherent(dd->parent,
					   in_nents * dma_sg_dsc_sz,
					   &dd->dma_sg_dsc_in, GFP_KERNEL | GFP_DMA);
	if (!dd->sg_dsc_in) {
		dev_err(dev, "dma_alloc_coherent() for input error\n");
		return 0;
	}

	dd->sg_dsc_out = dma_alloc_coherent(dd->parent,
					    out_nents * dma_sg_dsc_sz,
					    &dd->dma_sg_dsc_out, GFP_KERNEL | GFP_DMA);
	if (!dd->dma_sg_dsc_out) {
		dev_err(dev, "dma_alloc_coherent() for output error\n");
		dma_free_coherent(dd->parent, in_nents * dma_sg_dsc_sz,
				  dd->sg_dsc_in, dd->dma_sg_dsc_in);
		return 0;
	}
	in_sg = dd->in_sg;
	for (i = 0; i < in_nents; i++) {
		sg_dsc.dsc_cfg.d32 = 0;
		sg_dsc.dsc_cfg.b.valid = 1;
		sg_dsc.dsc_cfg.b.eoc = i == (in_nents - 1) ? 1 : 0;
		sg_dsc.dsc_cfg.b.length = in_sg->length;
		sg_dsc.addr = in_sg->dma_address;
		aml_dma_dsc_writer(dd->sg_dsc_in, i, &sg_dsc, 1, dd->dma);

		in_sg = sg_next(in_sg);
	}
	WARN_ON(in_sg);

	out_sg = dd->out_sg;
	for (i = 0; i < out_nents; i++) {
		sg_dsc.dsc_cfg.d32 = 0;
		sg_dsc.dsc_cfg.b.valid = 1;
		sg_dsc.dsc_cfg.b.eoc = i == (out_nents - 1) ? 1 : 0;
		sg_dsc.dsc_cfg.b.length = out_sg->length;
		sg_dsc.addr = out_sg->dma_address;
		aml_dma_dsc_writer(dd->sg_dsc_out, i, &sg_dsc, 1, dd->dma);
		out_sg = sg_next(out_sg);
	}
	WARN_ON(out_sg);

	aml_dma_link_debug(dd->sg_dsc_in, dd->dma_sg_dsc_in, in_nents,
			   __func__, dd->dma->dma_bus64);
	aml_dma_link_debug(dd->sg_dsc_out, dd->dma_sg_dsc_out, in_nents,
			   __func__, dd->dma->dma_bus64);

	dsc.src_addr = dd->dma_sg_dsc_in;
	dsc.tgt_addr = dd->dma_sg_dsc_out;
	dsc.length = total;
	dsc.dsc_cfg.d32 = 0;
	dsc.dsc_cfg.b.link_error = 1;
	aml_dma_dsc_writer(descriptor, 0, &dsc, 0, dd->dma);

	return total;
}

static struct aml_tdes_dev *aml_tdes_find_dev(struct aml_tdes_ctx *ctx)
{
	struct aml_tdes_dev *tdes_dd = NULL;
	struct aml_tdes_dev *tmp;

	spin_lock_bh(&aml_tdes.lock);
	if (!ctx->dd) {
		list_for_each_entry(tmp, &aml_tdes.dev_list, list) {
			tdes_dd = tmp;
			break;
		}
		ctx->dd = tdes_dd;
	} else {
		tdes_dd = ctx->dd;
	}

	spin_unlock_bh(&aml_tdes.lock);

	return tdes_dd;
}

static int aml_tdes_hw_init(struct aml_tdes_dev *dd)
{
	if (!(dd->flags & TDES_FLAGS_INIT)) {
		dd->flags |= TDES_FLAGS_INIT;
		dd->err = 0;
	}

	return 0;
}

static void aml_tdes_finish_req(struct aml_tdes_dev *dd, int err)
{
	unsigned long flags;

	spin_lock_irqsave(&dd->dma->dma_lock, flags);
	dd->flags &= ~TDES_FLAGS_BUSY;
	dd->dma->dma_busy &= ~DMA_FLAG_MAY_OCCUPY;
	spin_unlock_irqrestore(&dd->dma->dma_lock, flags);
}

static int aml_tdes_crypt_dma(struct aml_tdes_dev *dd, void *descriptor,
			      u32 nents)
{
	u32 op_mode = OP_MODE_ECB;
	struct dma_dsc_v2_64 dsc;
	u32 i = 0;
	int err = 0;
	u32 status = 0;

	if (dd->flags & TDES_FLAGS_CBC)
		op_mode = OP_MODE_CBC;

	for (i = 0; i < nents; i++) {
		/* some fields are already filled, read it back and update dsc */
		aml_dma_dsc_reader(descriptor, i, &dsc, 0, dd->dma);

		dsc.dsc_cfg.b.enc_sha_only = dd->flags & TDES_FLAGS_ENCRYPT;
		dsc.dsc_cfg.b.mode =
			((dd->ctx->keylen == DES_KEY_SIZE) ? MODE_DES :
			 ((dd->ctx->keylen == 2 * DES_KEY_SIZE) ?
			  MODE_TDES_2K : MODE_TDES_3K));
		dsc.dsc_cfg.b.op_mode = op_mode;
		dsc.dsc_cfg.b.eoc = (i == (nents - 1));
		dsc.dsc_cfg.b.owner = 1;

		aml_dma_dsc_writer(descriptor, i, &dsc, 0, dd->dma);
	}
	aml_dma_debug(descriptor, nents, __func__, dd->thread, dd->status, dd->dma);

	/* Start DMA transfer */
	dd->flags |= TDES_FLAGS_DMA;
	status = aml_dma_do_hw_crypto(dd->dma, descriptor, nents, dd->dma_descript_tab,
				      USE_BUSY_POLLING, DMA_FLAG_TDES_IN_USE);
	if (status & DMA_STATUS_KEY_ERROR)
		dd->flags |= TDES_FLAGS_ERROR;
	aml_dma_debug(descriptor, nents, "end tdes", dd->thread, dd->status, dd->dma);
	err = aml_tdes_crypt_dma_stop(dd);
	if (!err) {
		err = dd->flags & TDES_FLAGS_ERROR;
		dd->flags = (dd->flags & ~TDES_FLAGS_ERROR);
	}

	return err;
}

static int aml_tdes_crypt_dma_link_mode_start(struct aml_tdes_dev *dd)
{
	int err = 0;
	size_t count = 0;

	count = aml_tdes_sg_dma(dd, dd->descriptor, dd->total);
	dd->flags |= TDES_FLAGS_FAST;
	dd->fast_total = count;
	dd->total -= dd->fast_total;

	/* install IV */
	if ((dd->flags & TDES_FLAGS_CBC) && !(dd->flags & TDES_FLAGS_ENCRYPT))
		scatterwalk_map_and_copy(dd->req->iv,
					 dd->in_sg,
					 dd->fast_total - DES_BLOCK_SIZE,
					 DES_BLOCK_SIZE, 0);

	/* using 1 entry in link mode */
	err = aml_tdes_crypt_dma(dd, dd->descriptor, 1);

	return err;
}

static int aml_tdes_crypt_dma_start(struct aml_tdes_dev *dd)
{
	int err = 0;
	size_t count = 0;
	dma_addr_t addr_in, addr_out;
	struct dma_dsc_v2_64 dsc;
	u32 nents;

	/* slow dma */
	/* use cache buffers */
	count = aml_tdes_sg_copy(&dd->in_sg, &dd->in_offset,
				 dd->buf_in, dd->buflen, dd->total, 0);
	addr_in = dd->dma_addr_in;
	addr_out = dd->dma_addr_out;
	dd->dma_size = count;
	dma_sync_single_for_device(dd->parent, addr_in, dd->dma_size,
				   DMA_TO_DEVICE);

	dsc.src_addr = (u64)addr_in;
	dsc.tgt_addr = (u64)addr_out;
	dsc.length = count;
	dsc.dsc_cfg.d32 = 0;
	aml_dma_dsc_writer(dd->descriptor, 0, &dsc, 0, dd->dma);

	nents = 1;
	dd->flags &= ~TDES_FLAGS_FAST;

	dd->total -= count;
	err = aml_tdes_crypt_dma(dd, dd->descriptor, nents);

	return err;
}

static int aml_tdes_write_ctrl(struct aml_tdes_dev *dd)
{
	int err = 0;
	u32 *iv = NULL;

	err = aml_tdes_hw_init(dd);

	if (err)
		return err;

	if (dd->flags & TDES_FLAGS_CBC)
		iv = (u32 *)dd->req->iv;

	if (dd->ctx->kte >= 0)
		err = set_tdes_kl_key_iv(dd, dd->ctx->key, dd->ctx->keylen, iv);
	else
		err = set_tdes_key_iv(dd, dd->ctx->key, dd->ctx->keylen, iv);

	return err;
}

static int aml_tdes_handle_queue(struct aml_tdes_dev *dd,
				 struct skcipher_request *req)
{
	struct aml_tdes_ctx *ctx;
	struct aml_tdes_reqctx *rctx;
	s32 err = 0;

	/* assign new request to device */
	dd->req = req;
	dd->total = req->cryptlen;
	dd->in_offset = 0;
	dd->in_sg = req->src;
	dd->out_offset = 0;
	dd->out_sg = req->dst;

	rctx = skcipher_request_ctx(req);
	ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
	rctx->mode &= TDES_FLAGS_MODE_MASK;
	dd->flags = (dd->flags & ~TDES_FLAGS_MODE_MASK) | rctx->mode;
	dd->flags = (dd->flags & ~TDES_FLAGS_ERROR);
	dd->ctx = ctx;
	ctx->dd = dd;

	err = aml_tdes_write_ctrl(dd);
	if (!err) {
		if (dd->link_mode && !disable_link_mode &&
			!(dd->dma->dma_version == DMA_VERSION_1 &&
			dd->total >= MAX_DMA_BYTE_LENGTH)) {
			err = aml_tdes_crypt_dma_link_mode_start(dd);
		} else {
			do {
				err = aml_tdes_crypt_dma_start(dd);
			} while (!err && dd->total);
		}
	}
	if (err != -EINPROGRESS) {
		/* tdes_task will not finish it, so do it here */
		aml_tdes_finish_req(dd, err);
	}

	if (err)
		return -EAGAIN;
	else
		return 0;
}

int aml_tdes_process(struct skcipher_request *req)
{
	struct aml_tdes_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
	struct aml_tdes_dev *dd = tctx->dd;

	return aml_tdes_handle_queue(dd, req);
}

static int aml_tdes_crypt_dma_stop(struct aml_tdes_dev *dd)
{
	struct device *dev = dd->dev;
	int err = -EINVAL;
	size_t count;
	u32 in_nents = 0, out_nents = 0;
	u32 dma_sg_dsc_sz = aml_dma_get_dsc_sz(1, dd->dma);

	if (dd->flags & TDES_FLAGS_DMA) {
		err = 0;
		if  (dd->flags & TDES_FLAGS_FAST) {
			in_nents = sg_nents(dd->in_sg);
			out_nents = sg_nents(dd->out_sg);
			if (dd->in_sg != dd->out_sg) {
				dma_unmap_sg(dd->parent, dd->in_sg, in_nents,
					     DMA_TO_DEVICE);
				dma_unmap_sg(dd->parent, dd->out_sg, out_nents,
					     DMA_FROM_DEVICE);
			} else {
				dma_sync_sg_for_cpu(dd->parent, dd->in_sg,
						    in_nents, DMA_FROM_DEVICE);
				dma_unmap_sg(dd->parent, dd->in_sg,
					     in_nents, DMA_BIDIRECTIONAL);
			}

			aml_dma_link_debug(dd->sg_dsc_in, dd->dma_sg_dsc_in, in_nents,
					   __func__, dd->dma->dma_bus64);
			aml_dma_link_debug(dd->sg_dsc_out, dd->dma_sg_dsc_out, in_nents,
					   __func__, dd->dma->dma_bus64);

			dma_free_coherent(dd->parent, in_nents * dma_sg_dsc_sz,
					dd->sg_dsc_in, dd->dma_sg_dsc_in);
			dma_free_coherent(dd->parent, out_nents * dma_sg_dsc_sz,
					dd->sg_dsc_out, dd->dma_sg_dsc_out);
			/* install IV */
			if (dd->flags & TDES_FLAGS_CBC) {
				u32 length = dd->fast_total - DES_BLOCK_SIZE;

				if (dd->flags & TDES_FLAGS_ENCRYPT) {
					scatterwalk_map_and_copy(dd->req->iv,
								 dd->out_sg,
								 length,
								 DES_BLOCK_SIZE,
								 0);
				}
			}
		} else {
			dma_sync_single_for_cpu(dd->parent, dd->dma_addr_out,
						dd->dma_size, DMA_FROM_DEVICE);

			/* copy data */
			count = aml_tdes_sg_copy(&dd->out_sg, &dd->out_offset,
						 dd->buf_out, dd->buflen,
						 dd->dma_size, 1);
			if (count != dd->dma_size) {
				err = -EINVAL;
				dev_err(dev, "not all data converted: %zu\n",
					count);
			}
			/* install IV */
			if (dd->flags & TDES_FLAGS_CBC) {
				if (dd->flags & TDES_FLAGS_ENCRYPT) {
					memcpy(dd->req->iv, dd->buf_out +
					       dd->dma_size - DES_BLOCK_SIZE,
					       DES_BLOCK_SIZE);
				} else {
					memcpy(dd->req->iv, dd->buf_in +
					       dd->dma_size - DES_BLOCK_SIZE,
					       DES_BLOCK_SIZE);
				}
			}
		}
		dd->flags &= ~TDES_FLAGS_DMA;
	}

	return err;
}

static int aml_tdes_buff_init(struct aml_tdes_dev *dd)
{
	struct device *dev = dd->dev;
	int err = -ENOMEM;
	u32 dma_dsc_sz = aml_dma_get_dsc_sz(0, dd->dma);

	dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, 0);
	dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, 0);
	dd->buflen = PAGE_SIZE;
	dd->buflen &= ~(DES_BLOCK_SIZE - 1);

	if (!dd->buf_in || !dd->buf_out) {
		dev_err(dev, "unable to alloc pages.\n");
		goto err_alloc;
	}

	dd->descriptor =
		dmam_alloc_coherent(dd->parent,
				    MAX_NUM_TABLES * dma_dsc_sz,
				    &dd->dma_descript_tab, GFP_KERNEL | GFP_DMA);
	if (!dd->descriptor) {
		dev_err(dev, "dma descriptor error\n");
		err = -EINVAL;
		goto err_map_in;
	}

	/* MAP here */
	dd->dma_addr_in = dma_map_single(dd->parent, dd->buf_in,
					 dd->buflen, DMA_TO_DEVICE);
	if (dma_mapping_error(dd->parent, dd->dma_addr_in)) {
		dev_err(dev, "dma %zd bytes error\n", dd->buflen);
		err = -EINVAL;
		goto err_map_in;
	}

	dd->dma_addr_out = dma_map_single(dd->parent, dd->buf_out,
					  dd->buflen, DMA_FROM_DEVICE);
	if (dma_mapping_error(dd->parent, dd->dma_addr_out)) {
		dev_err(dev, "dma %zd bytes error\n", dd->buflen);
		err = -EINVAL;
		goto err_map_out;
	}

	return 0;

err_map_out:
	dma_unmap_single(dd->parent, dd->dma_addr_in, dd->buflen,
			 DMA_TO_DEVICE);
err_map_in:
	free_page((uintptr_t)dd->buf_out);
	free_page((uintptr_t)dd->buf_in);
err_alloc:
	if (err)
		dev_err(dev, "error: %d\n", err);
	return err;
}

static void aml_tdes_buff_cleanup(struct aml_tdes_dev *dd)
{
	u32 dma_dsc_sz = aml_dma_get_dsc_sz(0, dd->dma);

	dma_unmap_single(dd->parent, dd->dma_addr_out, dd->buflen,
			 DMA_FROM_DEVICE);
	dma_unmap_single(dd->parent, dd->dma_addr_in, dd->buflen,
			 DMA_TO_DEVICE);
	dmam_free_coherent(dd->parent, MAX_NUM_TABLES * dma_dsc_sz,
			dd->descriptor, dd->dma_descript_tab);
	free_page((unsigned long)dd->buf_out);
	free_page((unsigned long)dd->buf_in);
}

static int aml_tdes_crypt(struct skcipher_request *req, unsigned long mode)
{
	struct aml_tdes_ctx *ctx =
		crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
	struct aml_tdes_reqctx *rctx = skcipher_request_ctx(req);
	struct aml_tdes_dev *dd;
	int err;

	if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE)) {
		pr_err("request size is not exact amount of TDES blocks\n");
		return -EINVAL;
	}
	ctx->block_size = DES_BLOCK_SIZE;

	if (ctx->fallback && ctx->same_key) {
		struct skcipher_request *subreq = NULL;
		int ret = 0;

		subreq = kzalloc(sizeof(*subreq) +
				 crypto_skcipher_reqsize(ctx->fallback),
				 GFP_ATOMIC);
		if (!subreq)
			return -ENOMEM;

		skcipher_request_set_tfm(subreq, ctx->fallback);
		skcipher_request_set_callback(subreq, req->base.flags, NULL,
					      NULL);
		skcipher_request_set_crypt(subreq, req->src, req->dst,
					   req->cryptlen, req->iv);

		if (mode & TDES_FLAGS_ENCRYPT)
			ret = crypto_skcipher_encrypt(subreq);
		else
			ret = crypto_skcipher_decrypt(subreq);

		skcipher_request_free(subreq);
		return ret;
	}

	dd = aml_tdes_find_dev(ctx);
	if (!dd)
		return -ENODEV;

	rctx->mode = mode;

	if (pm_runtime_suspended(dd->parent)) {
		err = pm_runtime_get_sync(dd->parent);
		if (err < 0) {
			dev_err(dd->dev, "%s: pm_runtime_get_sync fails: %d\n",
				__func__, err);
			return err;
		}
	}
	return aml_dma_crypto_enqueue_req(dd->dma, &req->base);
}

static int aml_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
			  u32 keylen)
{
	int err;
	struct crypto_tfm *ctfm = crypto_skcipher_tfm(tfm);
	struct aml_tdes_ctx *ctx = crypto_skcipher_ctx(tfm);

	if (keylen != DES_KEY_SIZE) {
		return -EINVAL;
	}

	err = crypto_des_verify_key(ctfm, key);
	if (err != 0  && (ctfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
		ctfm->crt_flags |= CRYPTO_TFM_REQ_FORBID_WEAK_KEYS;
		return -EINVAL;
	}

	memcpy(ctx->key, key, keylen);
	ctx->keylen = keylen;
	ctx->kte = -1;

	return 0;
}

static int aml_des_kl_setkey(struct crypto_skcipher *tfm, const u8 *key,
			     u32 keylen)
{
	struct aml_tdes_ctx *ctx = crypto_skcipher_ctx(tfm);

	if (keylen != DES_KEY_SIZE) {
		return -EINVAL;
	}

	/* key[0:3] = kte */
	ctx->kte = *(uint32_t *)&key[0];
	if (ctx->kte < 0 || ctx->kte > MAX_KTE) {
		pr_err("%s: invalid kte: %d\n", __func__, ctx->kte);
		return -EINVAL;
	}

	ctx->keylen = keylen;

	return 0;
}

static int aml_tdes_ecb_encrypt(struct skcipher_request *req)
{
	return aml_tdes_crypt(req, TDES_FLAGS_ENCRYPT);
}

static int aml_tdes_ecb_decrypt(struct skcipher_request *req)
{
	return aml_tdes_crypt(req, 0);
}

static int aml_tdes_cbc_encrypt(struct skcipher_request *req)
{
	return aml_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CBC);
}

static int aml_tdes_cbc_decrypt(struct skcipher_request *req)
{
	return aml_tdes_crypt(req, TDES_FLAGS_CBC);
}

static int aml_tdes_lite_init_tfm(struct crypto_skcipher *tfm);
static void aml_tdes_lite_exit_tfm(struct crypto_skcipher *tfm);
static int aml_tdes_lite_setkey(struct crypto_skcipher *tfm, const u8 *key,
				unsigned int keylen);
static int aml_tdes_kl_setkey(struct crypto_skcipher *tfm, const u8 *key,
			      unsigned int keylen);
static struct skcipher_alg des_tdes_algs[] = {
	{
		.base.cra_name        = "ecb(des-aml)",
		.base.cra_driver_name = "ecb-des-aml",
		.base.cra_priority  =  100,
		.base.cra_flags     =  CRYPTO_ALG_ASYNC,
		.base.cra_blocksize =  DES_BLOCK_SIZE,
		.base.cra_ctxsize   =  sizeof(struct aml_tdes_ctx),
		.base.cra_alignmask =  0,
		.base.cra_module    =  THIS_MODULE,

		.init      =  aml_tdes_lite_init_tfm,
		.exit      =  aml_tdes_lite_exit_tfm,
		.min_keysize	=    DES_KEY_SIZE,
		.max_keysize	=    DES_KEY_SIZE,
		.setkey		=    aml_des_setkey,
		.encrypt	=    aml_tdes_ecb_encrypt,
		.decrypt	=    aml_tdes_ecb_decrypt,
	},
	{
		.base.cra_name        =  "cbc(des-aml)",
		.base.cra_driver_name =  "cbc-des-aml",
		.base.cra_priority  =  100,
		.base.cra_flags     =  CRYPTO_ALG_ASYNC,
		.base.cra_blocksize =  DES_BLOCK_SIZE,
		.base.cra_ctxsize   =  sizeof(struct aml_tdes_ctx),
		.base.cra_alignmask =  0,
		.base.cra_module    =  THIS_MODULE,

		.init      =  aml_tdes_lite_init_tfm,
		.exit      =  aml_tdes_lite_exit_tfm,
		.min_keysize	=    DES_KEY_SIZE,
		.max_keysize	=    DES_KEY_SIZE,
		.ivsize		=    DES_BLOCK_SIZE,
		.setkey		=    aml_des_setkey,
		.encrypt	=    aml_tdes_cbc_encrypt,
		.decrypt	=    aml_tdes_cbc_decrypt,
	},
	{
		.base.cra_name        = "ecb(des3_ede-aml)",
		.base.cra_driver_name = "ecb-tdes-aml",
		.base.cra_priority   = 100,
		.base.cra_flags      = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
		.base.cra_blocksize  = DES_BLOCK_SIZE,
		.base.cra_ctxsize    = sizeof(struct aml_tdes_ctx),
		.base.cra_alignmask  = 0,
		.base.cra_module     = THIS_MODULE,

		.init       = aml_tdes_lite_init_tfm,
		.exit       = aml_tdes_lite_exit_tfm,
		.min_keysize	=    2 * DES_KEY_SIZE,
		.max_keysize	=    3 * DES_KEY_SIZE,
		.setkey		=    aml_tdes_lite_setkey,
		.encrypt	=    aml_tdes_ecb_encrypt,
		.decrypt	=    aml_tdes_ecb_decrypt,
	},
	{
		.base.cra_name        = "cbc(des3_ede-aml)",
		.base.cra_driver_name = "cbc-tdes-aml",
		.base.cra_priority  = 100,
		.base.cra_flags      = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
		.base.cra_blocksize = DES_BLOCK_SIZE,
		.base.cra_ctxsize   = sizeof(struct aml_tdes_ctx),
		.base.cra_alignmask = 0,
		.base.cra_module    = THIS_MODULE,

		.init      = aml_tdes_lite_init_tfm,
		.exit      = aml_tdes_lite_exit_tfm,
		.min_keysize = 2 * DES_KEY_SIZE,
		.max_keysize = 3 * DES_KEY_SIZE,
		.ivsize	     = DES_BLOCK_SIZE,
		.setkey	     = aml_tdes_lite_setkey,
		.encrypt     = aml_tdes_cbc_encrypt,
		.decrypt     = aml_tdes_cbc_decrypt,
	},
	{
		.base.cra_name        = "ecb(des-kl-aml)",
		.base.cra_driver_name = "ecb-des-kl-aml",
		.base.cra_priority  =  100,
		.base.cra_flags     =  CRYPTO_ALG_ASYNC,
		.base.cra_blocksize =  DES_BLOCK_SIZE,
		.base.cra_ctxsize   =  sizeof(struct aml_tdes_ctx),
		.base.cra_alignmask =  0,
		.base.cra_module    =  THIS_MODULE,

		.init      =  aml_tdes_lite_init_tfm,
		.exit      =  aml_tdes_lite_exit_tfm,
		.min_keysize	=    DES_KEY_SIZE,
		.max_keysize	=    DES_KEY_SIZE,
		.setkey		=    aml_des_kl_setkey,
		.encrypt	=    aml_tdes_ecb_encrypt,
		.decrypt	=    aml_tdes_ecb_decrypt,
	},
	{
		.base.cra_name        =  "cbc(des-kl-aml)",
		.base.cra_driver_name =  "cbc-des-kl-aml",
		.base.cra_priority  =  100,
		.base.cra_flags     =  CRYPTO_ALG_ASYNC,
		.base.cra_blocksize =  DES_BLOCK_SIZE,
		.base.cra_ctxsize   =  sizeof(struct aml_tdes_ctx),
		.base.cra_alignmask =  0,
		.base.cra_module    =  THIS_MODULE,

		.init      =  aml_tdes_lite_init_tfm,
		.exit      =  aml_tdes_lite_exit_tfm,
		.min_keysize	=    DES_KEY_SIZE,
		.max_keysize	=    DES_KEY_SIZE,
		.ivsize		=    DES_BLOCK_SIZE,
		.setkey		=    aml_des_kl_setkey,
		.encrypt	=    aml_tdes_cbc_encrypt,
		.decrypt	=    aml_tdes_cbc_decrypt,

	},
	{
		.base.cra_name        = "ecb(tdes-kl-aml)",
		.base.cra_driver_name = "ecb-tdes-kl-aml",
		.base.cra_priority   = 100,
		.base.cra_flags      = CRYPTO_ALG_ASYNC,
		.base.cra_blocksize  = DES_BLOCK_SIZE,
		.base.cra_ctxsize    = sizeof(struct aml_tdes_ctx),
		.base.cra_alignmask  = 0,
		.base.cra_module     = THIS_MODULE,

		.init       = aml_tdes_lite_init_tfm,
		.exit       = aml_tdes_lite_exit_tfm,
		.min_keysize	=    2 * DES_KEY_SIZE,
		.max_keysize	=    3 * DES_KEY_SIZE,
		.setkey     =    aml_tdes_kl_setkey,
		.encrypt    =    aml_tdes_ecb_encrypt,
		.decrypt    =    aml_tdes_ecb_decrypt,
	},
	{
		.base.cra_name        = "cbc(tdes-kl-aml)",
		.base.cra_driver_name = "cbc-tdes-kl-aml",
		.base.cra_priority  = 100,
		.base.cra_flags     = CRYPTO_ALG_ASYNC,
		.base.cra_blocksize = DES_BLOCK_SIZE,
		.base.cra_ctxsize   = sizeof(struct aml_tdes_ctx),
		.base.cra_alignmask = 0,
		.base.cra_module    = THIS_MODULE,

		.init      = aml_tdes_lite_init_tfm,
		.exit      = aml_tdes_lite_exit_tfm,
		.min_keysize = 2 * DES_KEY_SIZE,
		.max_keysize = 3 * DES_KEY_SIZE,
		.ivsize	     = DES_BLOCK_SIZE,
		.setkey	     = aml_tdes_kl_setkey,
		.encrypt     = aml_tdes_cbc_encrypt,
		.decrypt     = aml_tdes_cbc_decrypt,
	}
};

static int aml_tdes_lite_init_tfm(struct crypto_skcipher *tfm)
{
	struct aml_tdes_ctx *ctx = crypto_skcipher_ctx(tfm);

	const char *alg_name = crypto_tfm_alg_name(&tfm->base);
	const char *driver_name = crypto_tfm_alg_driver_name(&tfm->base);
	const u32 flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK;
	char alg_to_use[16] = {0};

	crypto_skcipher_set_reqsize(tfm, sizeof(struct aml_tdes_reqctx));

	/* Driver name with '-kl' is only for installing key by kte.
	 * No need to allocate fallback for it
	 */
	if (!strstr(driver_name, "-kl")) {
		/* Allocate a fallback and abort if it failed. */
		strncpy(alg_to_use, "xxx(des3_ede)", sizeof(alg_to_use));
		memcpy(alg_to_use, alg_name, 3);
		ctx->fallback = crypto_alloc_skcipher(alg_to_use, 0,
						      flags);
		if (IS_ERR(ctx->fallback)) {
			pr_err("aml-tdes: fallback '%s' could not be loaded.\n",
			       alg_name);
			return PTR_ERR(ctx->fallback);
		}
	}

	return 0;
}

static void aml_tdes_lite_exit_tfm(struct crypto_skcipher *tfm)
{
	struct aml_tdes_ctx *ctx = crypto_skcipher_ctx(tfm);

	if (ctx->fallback)
		crypto_free_skcipher(ctx->fallback);

	ctx->fallback = NULL;
}

static int aml_tdes_kl_setkey(struct crypto_skcipher *tfm, const u8 *key,
			      unsigned int keylen)
{
	struct aml_tdes_ctx *ctx = crypto_skcipher_ctx(tfm);
	int ret = 0;

	if ((keylen != 2 * DES_KEY_SIZE) && (keylen != 3 * DES_KEY_SIZE)) {
		return -EINVAL;
	}

	/* key[0:3] = kte */
	ctx->kte = *(uint32_t *)&key[0];
	if (ctx->kte < 0 || ctx->kte > MAX_KTE) {
		pr_err("%s: invalid kte: %d\n", __func__, ctx->kte);
		return -EINVAL;
	}

	ctx->keylen = keylen;

	return ret;
}

static int aml_tdes_lite_setkey(struct crypto_skcipher *tfm, const u8 *key,
				unsigned int keylen)
{
	struct aml_tdes_ctx *ctx = crypto_skcipher_ctx(tfm);
	int ret = 0;
	u64 *tmp = NULL;

	if ((keylen != 2 * DES_KEY_SIZE) && (keylen != 3 * DES_KEY_SIZE)) {
		return -EINVAL;
	}

	memcpy(ctx->key, key, keylen);
	ctx->keylen = keylen;
	ctx->kte = -1;

	tmp = (u64 *)ctx->key;
	if (keylen == 2 * DES_KEY_SIZE)
		ctx->same_key = !(tmp[0] ^ tmp[1]);
	else
		ctx->same_key = !((tmp[0] ^ tmp[1]) | (tmp[0] ^ tmp[2]));

	if (ctx->same_key) {
		crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK);
		crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags &
					  CRYPTO_TFM_REQ_MASK);
		ret = crypto_skcipher_setkey(ctx->fallback, key, keylen);
	}

	return ret;
}

static struct skcipher_alg tdes_lite_algs[] = {
	{
		.base.cra_name        = "ecb(des3_ede-aml)",
		.base.cra_driver_name = "ecb-tdes-lite-aml",
		.base.cra_priority   = 100,
		.base.cra_flags      = CRYPTO_ALG_ASYNC |  CRYPTO_ALG_NEED_FALLBACK,
		.base.cra_blocksize  = DES_BLOCK_SIZE,
		.base.cra_ctxsize    = sizeof(struct aml_tdes_ctx),
		.base.cra_alignmask  = 0,
		.base.cra_module     = THIS_MODULE,

		.init       = aml_tdes_lite_init_tfm,
		.exit       = aml_tdes_lite_exit_tfm,
		.min_keysize	=    2 * DES_KEY_SIZE,
		.max_keysize	=    3 * DES_KEY_SIZE,
		.setkey     =    aml_tdes_lite_setkey,
		.encrypt    =    aml_tdes_ecb_encrypt,
		.decrypt    =    aml_tdes_ecb_decrypt,
	},
	{
		.base.cra_name        = "cbc(des3_ede-aml)",
		.base.cra_driver_name = "cbc-tdes-lite-aml",
		.base.cra_priority  = 100,
		.base.cra_flags     = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
		.base.cra_blocksize = DES_BLOCK_SIZE,
		.base.cra_ctxsize   = sizeof(struct aml_tdes_ctx),
		.base.cra_alignmask = 0,
		.base.cra_module    = THIS_MODULE,

		.init      = aml_tdes_lite_init_tfm,
		.exit      = aml_tdes_lite_exit_tfm,
		.min_keysize = 2 * DES_KEY_SIZE,
		.max_keysize = 3 * DES_KEY_SIZE,
		.ivsize	     = DES_BLOCK_SIZE,
		.setkey	     = aml_tdes_lite_setkey,
		.encrypt     = aml_tdes_cbc_encrypt,
		.decrypt     = aml_tdes_cbc_decrypt,
	},
	{
		.base.cra_name        = "ecb(tdes-kl-aml)",
		.base.cra_driver_name = "ecb-tdes-kl-aml",
		.base.cra_priority   = 100,
		.base.cra_flags      = CRYPTO_ALG_ASYNC |  CRYPTO_ALG_NEED_FALLBACK,
		.base.cra_blocksize  = DES_BLOCK_SIZE,
		.base.cra_ctxsize    = sizeof(struct aml_tdes_ctx),
		.base.cra_alignmask  = 0,
		.base.cra_module     = THIS_MODULE,

		.init       = aml_tdes_lite_init_tfm,
		.exit       = aml_tdes_lite_exit_tfm,
		.min_keysize	=    2 * DES_KEY_SIZE,
		.max_keysize	=    3 * DES_KEY_SIZE,
		.setkey     =    aml_tdes_kl_setkey,
		.encrypt    =    aml_tdes_ecb_encrypt,
		.decrypt    =    aml_tdes_ecb_decrypt,
	},
	{
		.base.cra_name        = "cbc(tdes-kl-aml)",
		.base.cra_driver_name = "cbc-tdes-kl-aml",
		.base.cra_priority  = 100,
		.base.cra_flags     = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
		.base.cra_blocksize = DES_BLOCK_SIZE,
		.base.cra_ctxsize   = sizeof(struct aml_tdes_ctx),
		.base.cra_alignmask = 0,
		.base.cra_module    = THIS_MODULE,

		.init      = aml_tdes_lite_init_tfm,
		.exit      = aml_tdes_lite_exit_tfm,
		.min_keysize = 2 * DES_KEY_SIZE,
		.max_keysize = 3 * DES_KEY_SIZE,
		.ivsize	     = DES_BLOCK_SIZE,
		.setkey	     = aml_tdes_kl_setkey,
		.encrypt     = aml_tdes_cbc_encrypt,
		.decrypt     = aml_tdes_cbc_decrypt,
	}
};

static void aml_tdes_unregister_algs(struct aml_tdes_dev *dd,
				     struct aml_tdes_info *tdes_info)
{
	int i;

	for (i = 0; i < tdes_info->num_algs; i++)
		crypto_unregister_skcipher(&tdes_info->algs[i]);
}

static int aml_tdes_register_algs(struct aml_tdes_dev *dd,
				  struct aml_tdes_info *tdes_info)
{
	int err, i, j;

	for (i = 0; i < tdes_info->num_algs; i++) {
		err = crypto_register_skcipher(&tdes_info->algs[i]);
		if (err)
			goto err_tdes_algs;
	}

	return 0;

err_tdes_algs:
	for (j = 0; j < i; j++)
		crypto_unregister_skcipher(&tdes_info->algs[j]);

	return err;
}

struct aml_tdes_info aml_des_tdes __initdata = {
	.algs = des_tdes_algs,
	.num_algs = ARRAY_SIZE(des_tdes_algs),
};

struct aml_tdes_info aml_tdes_lite __initdata = {
	.algs = tdes_lite_algs,
	.num_algs = ARRAY_SIZE(tdes_lite_algs),
};

#ifdef CONFIG_OF
static const struct of_device_id aml_tdes_dt_match[] = {
	{	.compatible = "amlogic,des_dma,tdes_dma",
		.data = &aml_des_tdes,
	},
	{	.compatible = "amlogic,tdes_dma",
		.data = &aml_tdes_lite,
	},
	{},
};
#else
#define aml_tdes_dt_match NULL
#endif

static int aml_tdes_probe(struct platform_device *pdev)
{
	struct aml_tdes_dev *tdes_dd;
	struct device *dev = &pdev->dev;
	int err = -EPERM;
	struct aml_tdes_info *tdes_info = NULL, *match;

	tdes_dd = devm_kzalloc(dev, sizeof(struct aml_tdes_dev), GFP_KERNEL);
	if (!tdes_dd) {
		err = -ENOMEM;
		goto tdes_dd_err;
	}

	match = (struct aml_tdes_info *)
		of_device_get_match_data(&pdev->dev);
	if (!match) {
		dev_err(dev, "%s: cannot find match dt\n", __func__);
		err = -EINVAL;
		goto tdes_dd_err;
	}

	tdes_info = devm_kzalloc(dev, sizeof(*tdes_info), GFP_KERNEL);
	if (!tdes_info) {
		err = -ENOMEM;
		goto tdes_dd_err;
	}
	tdes_info->algs = match->algs;
	tdes_info->num_algs = match->num_algs;
	tdes_dd->info = tdes_info;
	tdes_dd->dev = dev;
	tdes_dd->parent = dev->parent;
	tdes_dd->dma = dev_get_drvdata(dev->parent);
	tdes_dd->thread = tdes_dd->dma->thread;
	tdes_dd->status = tdes_dd->dma->status;
	tdes_dd->link_mode = tdes_dd->dma->link_mode;

	platform_set_drvdata(pdev, tdes_dd);

	INIT_LIST_HEAD(&tdes_dd->list);

	pm_runtime_use_autosuspend(dev);
	pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY);
	pm_runtime_enable(dev);

	err = pm_runtime_get_sync(dev);
	if (err < 0) {
		dev_err(dev, "%s: pm_runtime_get_sync fails: %d\n",
			__func__, err);
		goto err_tdes_pm;
	}

	err = aml_tdes_hw_init(tdes_dd);
	if (err)
		goto err_tdes_buff;

	err = aml_tdes_buff_init(tdes_dd);
	if (err)
		goto err_tdes_buff;

	spin_lock(&aml_tdes.lock);
	list_add_tail(&tdes_dd->list, &aml_tdes.dev_list);
	spin_unlock(&aml_tdes.lock);

	err = aml_tdes_register_algs(tdes_dd, tdes_info);
	if (err)
		goto err_algs;

	dev_dbg(dev, "Aml TDES_dma\n");

	pm_runtime_put_sync_autosuspend(dev);

	return 0;

err_algs:
	spin_lock(&aml_tdes.lock);
	list_del(&tdes_dd->list);
	spin_unlock(&aml_tdes.lock);
	aml_tdes_buff_cleanup(tdes_dd);
err_tdes_buff:
err_tdes_pm:
	pm_runtime_disable(dev);
tdes_dd_err:
	dev_err(dev, "initialization failed.\n");

	return err;
}

static void aml_tdes_remove(struct platform_device *pdev)
{
	static struct aml_tdes_dev *tdes_dd;
	struct aml_tdes_info *tdes_info = NULL;

	tdes_dd = platform_get_drvdata(pdev);
	if (!tdes_dd)
		return;

	tdes_info = tdes_dd->info;
	spin_lock(&aml_tdes.lock);
	list_del(&tdes_dd->list);
	spin_unlock(&aml_tdes.lock);

	aml_tdes_buff_cleanup(tdes_dd);
	aml_tdes_unregister_algs(tdes_dd, tdes_info);

	tasklet_kill(&tdes_dd->done_task);
	tasklet_kill(&tdes_dd->queue_task);

	pm_runtime_disable(tdes_dd->parent);
}

static struct platform_driver aml_tdes_driver = {
	.probe		= aml_tdes_probe,
	.remove		= aml_tdes_remove,
	.driver		= {
		.name	= "aml_tdes_dma",
		.owner	= THIS_MODULE,
		.of_match_table = aml_tdes_dt_match,
	},
};

int __init aml_tdes_driver_init(void)
{
	return platform_driver_register(&aml_tdes_driver);
}

void aml_tdes_driver_exit(void)
{
	platform_driver_unregister(&aml_tdes_driver);
}