BPI-CM9-Linux/aml-comp/kernel/aml-6.12/common_drivers/drivers/amfc/amfc.c

// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
/*
 * Copyright (c) 2019 Amlogic, Inc. All rights reserved.
 */
#include <linux/cdev.h>
#include <linux/types.h>
#include <linux/fs.h>
#include <linux/device.h>
#include <linux/slab.h>
#include <linux/delay.h>
#include <linux/uaccess.h>
#include <linux/sched.h>
#include <linux/platform_device.h>
#include <linux/of.h>
#include <linux/of_fdt.h>
#include <linux/of_device.h>
#include <linux/irqreturn.h>
#include <linux/memcontrol.h>
#include <linux/mmzone.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/crypto.h>
#include <linux/dma-mapping.h>
#include <linux/dma-map-ops.h>
#include <linux/completion.h>
#include <linux/clk.h>
#include <linux/pm_runtime.h>
#include <linux/pm_domain.h>
#include <linux/syscore_ops.h>
#include <linux/sched/clock.h>
#include <linux/ioctl.h>
#include <linux/vmalloc.h>
#include <linux/kthread.h>
#include <crypto/internal/scompress.h>

#ifdef CONFIG_HIGHMEM
#include <asm/fixmap.h>
#endif
#include <linux/cpu.h>
#include <linux/smp.h>
#include <linux/kallsyms.h>
#include <linux/of_irq.h>
#include <linux/interrupt.h>
#include <linux/amlogic/page_trace.h>
#include <linux/amlogic/cpu_version.h>
#include <linux/amlogic/gki_module.h>
#include <linux/amlogic/cpu_version.h>
#include <linux/amlogic/arm-smccc.h>
#include <linux/highmem.h>
#include <linux/amlogic/amfc_regs.h>
#include <linux/amlogic/amfc.h>
#include <linux/amlogic/user_fault.h>

#undef CONFIG_AMFC_TEST
#undef CONFIG_AMFC_DEBUG

static struct amfc *amfc;
static struct page *garbage_page;
static dev_t amfc_devno;
static struct cdev *amfc_cdev;

struct amfc *get_amfc_instance(void)
{
	return amfc;
}

static inline int _vmalloc_or_module_addr(const void *x)
{
	/*
	 * ARM, x86-64 and sparc64 put modules in a special place,
	 * and fall back on vmalloc() if that fails. Others
	 * just put it in the vmalloc space.
	 */
#if defined(CONFIG_MODULES) && defined(MODULES_VADDR)
	unsigned long addr = (unsigned long)kasan_reset_tag(x);

	if (addr >= MODULES_VADDR && addr < MODULES_END)
		return 1;
#endif

#ifdef CONFIG_HIGHMEM
	/* special area for kmap/kmap atomic */
	if ((unsigned long)x >= FIXADDR_START && (unsigned long)x < FIXADDR_END)
		return 1;
	if ((unsigned long)x >= PKMAP_ADDR(0) && (unsigned long)x < PKMAP_ADDR(LAST_PKMAP))
		return 1;
#endif
	return is_vmalloc_addr(x);
}

static unsigned int amfc_hw_read(unsigned int reg)
{
	return readl(amfc->io_base + reg);
}

static void amfc_hw_write(unsigned int value, unsigned int reg)
{
	writel(value, amfc->io_base + reg);
}

static unsigned int amfc_clk_read(unsigned int reg)
{
	return readl(amfc->clk_base + reg);
}

/* MHz */
static unsigned int amfc_clk[] = {
	 24000000,
	166666666,
	100000000,
	800000000,
	666666666,
	500000000,
	400000000,
	576000000,
	285000000
};

static unsigned int init_clk;

static int early_amfc_clk_set(char *buf)
{
	int clk = 0;
	int i = 0;

	if (kstrtoint(buf, 10, &clk))
		return -EINVAL;

	for (i = 0; i < ARRAY_SIZE(amfc_clk); i++) {
		if (clk == amfc_clk[i]) {
			pr_emerg("set amfc_clk to %d Hz\n", clk);
			init_clk = clk;
			return 1;
		}
	}
	return -EINVAL;
}
__setup("amfc_clk=", early_amfc_clk_set);

static unsigned int default_log;

static int early_amfc_log_set(char *buf)
{
	int i = 0;

	if (kstrtoint(buf, 10, &i))
		return -EINVAL;

	pr_emerg("set amfc_log to %d\n", i);
	default_log = i;
	return 1;
}
__setup("amfc_log=", early_amfc_log_set);


static int amfc_hw_init(void)
{
	unsigned int value;

	if (init_clk)
		amfc->rate = init_clk;
	/* fix 500mhz for s7d reva */

	amfc->hw_version = amfc_hw_read(AMFC_GL_VERSION);
	pr_info("AMFC VLSI version:%x, feature:%x, clk:%ld\n",
		amfc->hw_version,
		amfc_hw_read(AMFC_GL_CMD0_FEATURE), amfc->rate);
	clk_set_rate(amfc->clk, amfc->rate);

	/* sw reset */
	amfc_hw_write(0x80000000, AMFC_GL_CMD0_CONTROL);
	amfc_hw_write(0, AMFC_GL_CMD0_CONTROL);
	amfc_hw_write(0, AMFC_GL_CMD1_CONTROL);
	amfc_hw_write(3, AMFC_GL_CMD0_IRQCLR);
	amfc_hw_write(3, AMFC_GL_CMD1_IRQCLR);
	amfc_hw_write(0, AMFC_GL_MISC);		// cmd0/1 mixed
	amfc_hw_write((2 << 6) | (2 << 4) | (1 << 1) | (1), AMFC_CODEC_CTRL);
	amfc_hw_write(1, AMFC_ZSTD_MODE_MISC);

	value = amfc_hw_read(AMFC_WR_MIF_CTRL);
	value &= ~(0x7 << 8);
	value |=  (2 << 8);
	amfc_hw_write(value, AMFC_WR_MIF_CTRL);

	value = amfc_hw_read(AMFC_RD_MIF_CTRL);
	value &= ~(0x7 << 8);
	value |=  (2 << 8);
	amfc_hw_write(value, AMFC_RD_MIF_CTRL);

	return 0;
}

static void fix_amfc_clock(struct platform_device *pdev)
{
	struct resource *res;
	void *license;
	unsigned int package, reg;
	if (!is_meson_t6d_cpu())
		return;
	package = get_meson_cpu_version(MESON_CPU_VERSION_LVL_PACK);
	pr_info("T6D, package:%d\n", package);
	if (package == 2) { /* t950d5z */
		amfc->rate = 666666666;
		return;
	}
	res = platform_get_resource(pdev, IORESOURCE_MEM, 2);
	if (res) {
		license = ioremap(res->start, resource_size(res));
		if (license) {
			reg = readl(license);
			pr_info("license:%x\n", reg);
			if (reg & BIT(20))
				amfc->rate = 666666666;
			iounmap(license);
		}
	}
}

static unsigned long vmalloc_to_phys(void *va, int check)
{
	unsigned long pfn = vmalloc_to_pfn(va);

	if (check)
		WARN_ON(!pfn_valid(pfn));
	return __pa(pfn_to_kaddr(pfn)) + offset_in_page(va);
}

inline static unsigned long get_cache_line_size(void)
{
	unsigned long ctr_el0 = 0;

	asm volatile (
	#ifdef CONFIG_ARM64
		"mrs	%[ctr_el0], ctr_el0			\n"
	#else		// ARM
		"mrc	p15, 0, %[ctr_el0], cr0, cr0, 1		\n"
	#endif
		: [ctr_el0] "+r" (ctr_el0)
		:
		: "memory", "cc"
	);
	return ctr_el0;
}

static void amfc_map_addr(unsigned long addr, long size, int dir)
{
	unsigned long ctr_el0 = get_cache_line_size();
	unsigned long offset;

	// cache line size
	ctr_el0 = ((ctr_el0 >> 16) & 0xf) << 4;
	offset = addr & (ctr_el0 - 1);
	if (offset) { // align down to cache line
		size += offset;
		addr -= offset;
	}

	if (dir & DMA_FROM_DEVICE) {	// invalid
		while (size > 0) {
			asm volatile (
			#ifdef CONFIG_ARM64
				"dc	civac, %[addr]			\n"
			#else
				" mcr	p15, 0, %[addr], c7, c6, 1	\n"
			#endif
				:
				: [addr] "r" (addr)
				: "memory", "cc"
			);
			addr += ctr_el0;
			size -= ctr_el0;
		}
	} else {
		while (size > 0) {	// clean
			asm volatile (
			#ifdef CONFIG_ARM64
				"dc	cvac, %[addr]		\n"
			#else
				" mcr	p15, 0, %[addr], c7, c10, 1	\n"
			#endif
				:
				: [addr] "r" (addr)
				: "memory", "cc"
			);
			addr += ctr_el0;
			size -= ctr_el0;
		}
	}
#ifdef CONFIG_ARM64
	dsb(sy);
	dmb(sy);
#else
	dsb();
	dmb();
#endif
}

static void amfc_unmap_addr(unsigned long addr, long size, int dir)
{
	unsigned long ctr_el0 = get_cache_line_size();
	unsigned long offset;

	// cache line size
	ctr_el0 = ((ctr_el0 >> 16) & 0xf) << 4;
	offset = addr & (ctr_el0 - 1);
	if (offset) {
		size += offset;
		addr -= offset;
	}

	if (dir & DMA_FROM_DEVICE) {	// invalid
		while (size > 0) {
			asm volatile (
			#ifdef CONFIG_ARM64
				"dc	civac, %[addr]			\n"
			#else
				" mcr	p15, 0, %[addr], c7, c6, 1	\n"
			#endif
				:
				: [addr] "r" (addr)
				: "memory", "cc"
			);
			addr += ctr_el0;
			size -= ctr_el0;
		}
	} else {
	#if 0
		while (addr < size) {	// clean
			asm volatile (
			#ifdef CONFIG_ARM64
				"dc	cvac, %[addr]		\n"
			#else
				" mcr	p15, 0, %[addr], c7, c10, 1	\n"
			#endif
				:
				: [addr] "r" (addr)
				: "memory", "cc"
			);
			addr += ctr_el0;
		}
	#endif
	}
#ifdef CONFIG_ARM64
	dsb(sy);
	dmb(sy);
#else
	dsb();
	dmb();
#endif
}

static int show_regs(char *buf)
{
	unsigned int *p;
	unsigned int phy_base, i, size  = 0;

	if (!amfc->log && !buf)
		return 0;

	phy_base = vmalloc_to_phys(amfc->io_base, 0);
	p = (unsigned int *)amfc->io_base;
	if (buf) {
		size += sprintf(buf + size, "AMFC_REGS:");
		for (i = 0; i < 0x48; i++) {
			if (!(i & 0x3))
				size += sprintf(buf + size, "\n0x%08x: ",
						phy_base + i * 4);
			size += sprintf(buf + size, "%08x ", p[i]);
		}
		size += sprintf(buf + size, "\n");
		size += sprintf(buf + size, "CLKCTRL_AMFC_CLK_CTRL:%08x\n",
				amfc_clk_read(CLKCTRL_AMFC_CLK_CTRL));
		size += sprintf(buf + size, "AMFC_SECURE_TOP_REG:%08x\n",
				amfc_hw_read(AMFC_SECURE_TOP_REG));
	} else {
		pr_info_ratelimited("AMFC_REGS:\n");
		for (i = 0; i < 0x48; i++) {
			if (!(i & 0x3))
				pr_info_ratelimited("0x%08x: ", phy_base + i * 4);
			pr_cont("%08x ", p[i]);
		}
		pr_info_ratelimited("\n");
		pr_info_ratelimited("CLKCTRL_AMFC_CLK_CTRL:%08x\n",
			amfc_clk_read(CLKCTRL_AMFC_CLK_CTRL));
		pr_info_ratelimited("AMFC_SECURE_TOP_REG:%08x\n",
			amfc_hw_read(AMFC_SECURE_TOP_REG));
	}
	return size;
}

static void show_acl(struct amfc_cmd_list *acl)
{
	if (!acl || !amfc->log)
		return;

	pr_info_ratelimited("acl:%px\n", acl);
	pr_info_ratelimited("%08x %08x %08x %08x  %08x %08x %08x %08x\n",
		acl->src_addr, acl->dst_addr, acl->link_addr, acl->src_size,
		acl->dst_size, acl->control,  acl->status,    acl->result_size);
}

#ifndef CONFIG_AMFC_DEBUG
static void dump_addr(void *buf, unsigned int size)
{
	int i;
	unsigned int *p = (unsigned int *)buf;

	if (!amfc->log)
		return ;

	pr_info_ratelimited("%s addr:%px, size:%d\n", __func__, buf, size);
	for (i = 0; i < size / 4; i += 4) {
		pr_info_ratelimited("%px: %08x %08x %08x %08x\n",
			p + i, p[i], p[i + 1], p[i + 2], p[i + 3]);
	}
}
#else
inline static void dump_addr(void *buf, unsigned int size)
{
}
#endif

/*
 *TODO:
 * 1, add debug fs
 * 2, lock protect for multi-processor
 */
static void *build_tables(unsigned int *table, void *base, ssize_t size, int stream)
{
	int i, count;
	unsigned long tmp, pfn, end = (unsigned long)base + size - 1;
	struct page *page;

	if (!table)
		return NULL;

	if (((unsigned long)base & ~PAGE_MASK) && amfc->hw_version < AMFC_VER_1_1) {
		pr_emerg("%s, base:%px not align to %ld\n", __func__, base, PAGE_SIZE);
		WARN_ON(1);
		return NULL;
	}

	/* physical contiguous in uboot */
	count = (end >> PAGE_SHIFT) - (((unsigned long)base) >> PAGE_SHIFT) + 1;
	for (i = 0; i < count; i++) {
		if (_vmalloc_or_module_addr(base)) {
			page = vmalloc_to_page(base);
			if (!page) { /* out of vmalloc range due to a new cache line added */
				pfn = page_to_pfn(garbage_page);
			} else {
				pfn = page_to_pfn(page);
			}
		} else {
			pfn = virt_to_pfn(base);
		}
		tmp = (pfn << 6) | 1;
		table[i] = tmp;
	#ifdef CONFIG_AMFC_DEBUG
		pr_info("%s, base:%px, table:%px:%x, pfn:%lx\n",
			__func__, base, table, tmp, pfn);
	#endif
		base += PAGE_SIZE;
	}
	if (stream) {
		/* avoid overflow if we force exit */
		pfn = page_to_pfn(garbage_page);
		tmp = (pfn << 6) | 1;
		table[count++] = tmp;
	}
	table[count++] = 0;	// hardware stop
	amfc_map_addr((long)table, count * 4, DMA_TO_DEVICE);

	return table;
}

/* build table address for user space */
static void *build_user_tables(unsigned int *table, void *base, ssize_t size, int stream)
{
	int i, count;
	unsigned long tmp, pfn, end = (unsigned long)base + size - 1;
	struct mm_struct *mm;

	mm = current->mm;

	if (!mm)
		return NULL;

	if (((unsigned long)base & ~PAGE_MASK) && amfc->hw_version < AMFC_VER_1_1) {
		pr_emerg("%s, base:%px not align to %ld\n", __func__, base, PAGE_SIZE);
		WARN_ON(1);
		return NULL;
	}

	/* physical contiguous in uboot */
	count = (end >> PAGE_SHIFT) - (((unsigned long)base) >> PAGE_SHIFT) + 1;
	for (i = 0; i < count; i++) {
		pfn = get_user_pfn(mm, (unsigned long)base);
		tmp = (pfn << 6) | 1;
		table[i] = tmp;
	#ifdef CONFIG_AMFC_DEBUG
		pr_info("%s, base:%px, table:%px:%x, pfn:%lx\n",
			__func__, base, table, tmp, pfn);
	#endif
		base += PAGE_SIZE;
	}
	if (stream) {
		/* avoid overflow if we force exit */
		pfn = page_to_pfn(garbage_page);
		tmp = (pfn << 6) | 1;
		table[count++] = tmp;
	}
	table[count++] = 0;	// hardware stop
	amfc_map_addr((long)table, count * 4, DMA_TO_DEVICE);

	return table;
}

inline static int get_page_count(size_t size)
{
	int num;

	size = PAGE_ALIGN(size);
	num  = size / PAGE_SIZE * sizeof(int);
	return PAGE_ALIGN(num) / PAGE_SIZE;
}

static struct page *alloc_page_table(size_t size, int type, int *order)
{
	struct page *page;
	unsigned long pages;

	pages = get_page_count(size);
	if (*order == 0) {
		if (likely(pages == 1))
			return amfc->pages[type];
	}

	*order = get_order(pages << PAGE_SHIFT);
	page = alloc_pages(GFP_KERNEL, *order);
	return page;
}

static void set_up_addr(struct amfc_cmd_list *acl, unsigned long addr, unsigned int off,
			int type, int isvmalloc)
{
	unsigned int low, high = 0;

	low  = addr & 0xffffffff;
#ifdef CONFIG_64BIT
	high = (addr >> 32) & 0xf;
#endif
	switch (type) {
	case ADDR_SRC:
		acl->src_addr = low;
		acl->src_addr_h = high;
		if (amfc->hw_version > AMFC_VER_1_0)
			acl->link_addr |= (off << 16);
		break;

	case ADDR_DST:
		acl->dst_addr = low;
		acl->dst_addr_h = high;
		if (amfc->hw_version > AMFC_VER_1_0)
			acl->link_addr |= (off);
		break;

	default:
		break;
	}
}

/* map for physical scatter address */
static void *create_map(struct amfc_cmd_list *acl, void *addr,
			ssize_t size, int stream, int type, int *order)
{
	struct page *page_table = NULL;
	void *table_addr = NULL;
	int addr_type = -1;
	int need_scatter = 1;

	if (stream)
		size -= AMFC_STREAM_MARGIN;

	/* start and end in same page */
	if ((((unsigned long)addr + size - 1) >> PAGE_SHIFT) ==
	    ((unsigned long)addr >> PAGE_SHIFT) && !stream)
		need_scatter = 0;

	if (need_scatter) {
		page_table = alloc_page_table(size, type, order);
		if (!page_table)
			return ERR_PTR(-ENOMEM);

		table_addr = build_tables(page_address(page_table), addr, size, stream);
		if (!table_addr)
			return ERR_PTR(-EINVAL);
	}

	switch (type) {
	case TABLE_SRC_DECOMPRESS:
	case TABLE_SRC_COMPRESS:
		addr_type = ADDR_SRC;
		break;

	case TABLE_DST_DECOMPRESS:
	case TABLE_DST_COMPRESS:
		addr_type = ADDR_DST;
		break;

	default:
		WARN_ON(1);
		break;
	}
	if (need_scatter) {
		set_up_addr(acl, page_to_phys(page_table), PAGE_OFF(addr), addr_type, 1);
		if (addr_type == ADDR_SRC)
			acl->src_scatter = 1;
		else if (addr_type == ADDR_DST)
			acl->dst_scatter = 1;
		return page_table;
	}
	set_up_addr(acl, vmalloc_to_phys(addr, 1), PAGE_OFF(addr), addr_type, 1);
	return NULL;
}

/* map for user address, always use scatter */
static void *create_user_map(struct amfc_cmd_list *acl, void *addr,
			     ssize_t size, int type, int *order)
{
	struct page *page_table = NULL;
	void *table_addr = NULL;
	int addr_type = -1;

	page_table = alloc_page_table(size, type, order);
	if (!page_table)
		return ERR_PTR(-ENOMEM);

	table_addr = build_user_tables(page_address(page_table), addr, size, 1);
	if (!table_addr)
		return ERR_PTR(-EINVAL);

	switch (type) {
	case TABLE_SRC_DECOMPRESS:
	case TABLE_SRC_COMPRESS:
		addr_type = ADDR_SRC;
		break;

	case TABLE_DST_DECOMPRESS:
	case TABLE_DST_COMPRESS:
		addr_type = ADDR_DST;
		break;

	default:
		WARN_ON(1);
		break;
	}
	set_up_addr(acl, page_to_phys(page_table), PAGE_OFF(addr), addr_type, 1);
	if (addr_type == ADDR_SRC)
		acl->src_scatter = 1;
	else if (addr_type == ADDR_DST)
		acl->dst_scatter = 1;
	return page_table;
}

int amfc_probe_done(void)
{
	if (amfc)
		return amfc->init_done;
	else
		return 0;
}
EXPORT_SYMBOL(amfc_probe_done);

/*
 * src: source buffer that need decompress
 * dst: destination buffer that need store
 * src_size: size of source buffer
 * dst_size: size for destination buffer, caller should make sure
 *           dst size is enough
 * stream: stream decompress mode
 * return value: decompressed size if success, negative value if failed
 */
int amfc_decompress(void *src, void *dst, ssize_t src_size, ssize_t dst_size, int stream)
{
	struct page *src_table = NULL, *dst_table = NULL;
	struct amfc_cmd_list *acl;
	int ret = -EINVAL;
	int src_order = 0, dst_order = 0;
	unsigned int status, control, clks = 0;
	unsigned long flags;
	unsigned long long timeout = ((src_size) / PAGE_SIZE) * 50 + 5000; // us
	unsigned long long tick, cur;
	int need_copy = 0;
	void *tmp;

	if (!amfc || !amfc->decompress)
		return -ENOMEM;

	if (!src || !dst || !src_size || !dst_size)
		return -EINVAL;

	if (atomic_read(&amfc->in_suspend))
		return -ENODEV;

	if (amfc->log > 2)
		pr_info("%s, src:%px, dst:%px, src size:%d, dst size:%d\n",
			__func__, src, dst, (int)src_size, (int)dst_size);
	if (stream)
		amfc_map_addr((long)dst, dst_size - AMFC_STREAM_MARGIN, DMA_FROM_DEVICE);
	else
		amfc_map_addr((long)dst, dst_size, DMA_FROM_DEVICE);
	amfc_map_addr((long)src, src_size, DMA_TO_DEVICE);

	if (spin_is_locked(&amfc->dec_lock))
		amfc->d_congestion++;

	if (amfc->work_mode == AMFC_POLL_MODE)
		spin_lock(&amfc->dec_lock);
	else
		spin_lock_irqsave(&amfc->dec_lock, flags);

	amfc->d_count++;
	if ((amfc->hw_version < AMFC_VER_1_1) && ((unsigned long)dst & ~PAGE_MASK) &&
	    (dst_size + ((unsigned long)dst & ~PAGE_MASK) > PAGE_SIZE)) {
		tmp = dst;
		dst = amfc->bounce_buffer;
		need_copy   = 1;
	}

	acl = amfc->decompress;
	/* setup command list, decompress use cmd0 */
	memset(acl, 0, sizeof(struct amfc_cmd_list));
	acl->src_size = src_size;
	acl->dst_size = dst_size;
	if (_vmalloc_or_module_addr(src)) {
		src_table = create_map(acl, src, src_size, 0,
				       TABLE_SRC_DECOMPRESS, &src_order);
		if (IS_ERR(src_table))
			goto out;
	} else if ((unsigned long)src < TASK_SIZE) {
		src_table = create_user_map(acl, src, src_size,
				       TABLE_SRC_DECOMPRESS, &src_order);
		if (IS_ERR(src_table))
			goto out;
	} else {
		set_up_addr(acl, virt_to_phys(src), PAGE_OFF(src), ADDR_SRC, 0);
	}

	if (_vmalloc_or_module_addr(dst) || stream) {
		dst_table = create_map(acl, dst, dst_size, stream,
				       TABLE_DST_DECOMPRESS, &dst_order);
		if (IS_ERR(dst_table))
			goto out;
	} else if ((unsigned long)dst < TASK_SIZE) {
		dst_table = create_user_map(acl, dst, dst_size,
				       TABLE_DST_DECOMPRESS, &dst_order);
		if (IS_ERR(dst_table))
			goto out;
	} else {
		set_up_addr(acl, virt_to_phys(dst), PAGE_OFF(dst), ADDR_DST, 0);
	}
again:
	control        = 1;
	acl->algorithm = ALGORITHM_ZSTD;
	acl->end       = 1;
	acl->compress  = CMD_DECOMPRESS;
	acl->owner     = 1;
	acl->status    = 0xffffffff;
	if (!amfc->work_mode) {
		//acl->interrupt = 1;
		control |= (1 << 4);
	}

	amfc_map_addr((long)acl, sizeof(*acl), DMA_TO_DEVICE);

	amfc_hw_write(3, AMFC_GL_CMD1_IRQCLR);
	amfc_hw_write(virt_to_phys(acl) >> ADDR_SHIFT, AMFC_GL_CMD1_DESC_BASE_ADDR);
	/* trigger with irq en */
	amfc_hw_write(control, AMFC_GL_CMD1_CONTROL);

	if (!amfc->work_mode) {
		/* wait hw irq */
		ret = wait_for_completion_interruptible(&amfc->dcomp);
		if (ret) {
			ret = -EINTR;
			goto out;
		}
	} else {
		/* poll */
		tick = sched_clock();
		while (1) {
			status = amfc_hw_read(AMFC_GL_CMD1_STATUS);
			if ((status & IRQ_MASK))
				break;
			cur = sched_clock();
			if (cur - tick >= timeout * 1000) {
				pr_emerg_ratelimited("%s timeout:%lld -> %lld, %lld\n",
					 __func__, tick, cur, timeout);
				show_regs(NULL);
				amfc_unmap_addr((long)acl, sizeof(*acl), DMA_FROM_DEVICE);
				show_acl(acl);
				pr_info_ratelimited("src:%px, dst:%px, src_size:%d, dst_size:%d\n", src, dst, (int)src_size, (int)dst_size);
				dump_addr(src, src_size);
				if (cur - tick >= timeout * 500UL)
					break;
				// init again and retry;
				amfc_hw_init();
				/* release for irq */
				local_irq_enable();
				local_irq_disable();
				goto again;
			}
			if (atomic_read(&amfc->in_suspend)) {
				ret = -ENODEV;
				goto out;
			}
		}
	}
	clks = amfc_hw_read(AMFC_CMD1_TIME_MEASURE);

	amfc_unmap_addr((long)acl, sizeof(*acl), DMA_FROM_DEVICE);
	status = amfc_hw_read(AMFC_GL_CMD1_STATUS);
	if (!(status & AMFC_ERROR_MASK)) {
		ret = acl->result_size;
	} else {
		ret = 0 - ((status & AMFC_ERROR_MASK) >> 8);
	}
out:
	if (!IS_ERR_OR_NULL(src_table) && src_table != amfc->pages[TABLE_SRC_DECOMPRESS])
		__free_pages(src_table, src_order);
	if (!IS_ERR_OR_NULL(dst_table) && dst_table != amfc->pages[TABLE_DST_DECOMPRESS])
		__free_pages(dst_table, dst_order);
	if (atomic_read(&amfc->in_suspend)) /* check again */
		ret = -ENODEV;
	if (ret < 0) {
		if (ret == -ENODEV) {
			pr_err("%s, decompress failed caused by suspend\n", __func__);
		} else {
			amfc->in_dec_err = 1;
			if ((ret == -AMFC_DEC_DST_SIZE_OVF || ret == -AMFC_DEC_DST_PAGE_ERR) && stream) {
				while (amfc_hw_read(AMFC_WR_MIF_STATUS))
					;
				if (amfc->log)
					pr_info("decompress acl:%px, src:%px, dst:%px, src size:%5d, dstsize:%5d result size:%5d:%5d, tick:%d\n",
						acl, src, dst, (int)src_size, (int)dst_size, ret,
						acl->result_size, clks);
			} else {
				pr_err_ratelimited("acl:%px, decompress failed, src:%px, dst:%px, ssize:%d, dsize:%d, ret:%d, status:%x\n",
					acl, src, dst, (int)src_size, (int)dst_size,
					ret, amfc_hw_read(AMFC_GL_CMD1_STATUS));
				show_regs(NULL);
				show_acl(acl);
				if (!IS_ERR_OR_NULL(src_table))
					dump_addr(page_address(src_table), 128);
				if (!IS_ERR_OR_NULL(dst_table))
					dump_addr(page_address(dst_table), 128);
				dump_addr(src, src_size);
				dump_addr(dst, dst_size);
				need_copy = 0;	// decompress real failed
			}
			/* sw reset */
			while (!spin_trylock(&amfc->com_lock)) {
				if (amfc->in_enc_err) {
					amfc_hw_write(0x80000000, AMFC_GL_CMD1_CONTROL);
					goto error_handled;
				}
			}
			amfc_hw_write(0x80000000, AMFC_GL_CMD1_CONTROL);
			spin_unlock(&amfc->com_lock);
error_handled:
			amfc->in_dec_err = 0;
		}
	} else {
		amfc->din += src_size;
		if (stream) {  // separate for EROFS and ZRAM
			amfc->e_dout      += ret;
			amfc->e_dtick     += clks;
			amfc->total_dtick += clks;
		} else {
			amfc->z_dout      += ret;
			amfc->z_dtick     += clks;
			amfc->total_dtick += clks;
		}
		if (amfc->log > 2)
			pr_info("decompress ACL:%px, src:%px, dst:%px, src size:%5d, dst size,%5d, result size:%5d, tick:%d\n",
				acl, src, dst, (int)src_size, (int)dst_size, ret, clks);
	}
	amfc_hw_write(0x03, AMFC_GL_CMD1_IRQCLR);
	if (need_copy)
		memcpy(tmp, amfc->bounce_buffer, dst_size - AMFC_STREAM_MARGIN);

	if (amfc->work_mode == AMFC_POLL_MODE)
		spin_unlock(&amfc->dec_lock);
	else
		spin_unlock_irqrestore(&amfc->dec_lock, flags);

	if (stream)
		amfc_unmap_addr((long)dst, dst_size - AMFC_STREAM_MARGIN, DMA_FROM_DEVICE);
	else
		amfc_unmap_addr((long)dst, dst_size, DMA_FROM_DEVICE);

	return ret;
}
EXPORT_SYMBOL(amfc_decompress);


/*
 * src: source buffer that need compress
 * dst: destination buffer that need store
 * src_size: size of source buffer
 * dst_size: size for destination buffer, caller should make sure
 *           dst size is enough
 * return value: compressed size if success, negative value if failed
 */
int amfc_compress(void *src, void *dst, ssize_t src_size, ssize_t dst_size)
{
	struct page *src_table = NULL, *dst_table = NULL;
	struct amfc_cmd_list *acl;
	int ret = -EINVAL;
	int src_order = 0, dst_order = 0;
	unsigned int status = 0, control, clks = 0;
	unsigned long flags;
	unsigned long long timeout = ((src_size) / PAGE_SIZE) * 100 + 5000; // us
	unsigned long long tick, cur;

	if (!amfc || !amfc->compress)
		return -ENOMEM;

	if (!src || !dst || !src_size || !dst_size)
		return -EINVAL;

	if (atomic_read(&amfc->in_suspend))
		return -ENODEV;

	if (amfc->log > 2)
		pr_info("%s, src:%px, dst:%px, src size:%d, dst size:%d\n",
			__func__, src, dst, (int)src_size, (int)dst_size);
	amfc_map_addr((long)src, src_size, DMA_TO_DEVICE);
	/* for compress, zram usually give size large than source, but we only
	 * need flush source size
	 */
	amfc_map_addr((long)dst, dst_size > (src_size + CACHELINE_SIZE) ? (src_size + CACHELINE_SIZE) : dst_size, DMA_FROM_DEVICE);
	if (spin_is_locked(&amfc->com_lock))
		amfc->c_congestion++;

	if (amfc->work_mode == AMFC_POLL_MODE)
		spin_lock(&amfc->com_lock);
	else
		spin_lock_irqsave(&amfc->com_lock, flags);
	amfc->c_count++;
	acl = amfc->compress;
	/* setup command list, decompress use cmd0 */
	memset(acl, 0, sizeof(struct amfc_cmd_list));
	acl->src_size = src_size;
	acl->dst_size = dst_size;
	if (_vmalloc_or_module_addr(src)) {
		src_table = create_map(acl, src, src_size, 0,
				       TABLE_SRC_COMPRESS, &src_order);
		if (IS_ERR(src_table))
			goto out;
	} else if ((unsigned long)src < TASK_SIZE) {
		src_table = create_user_map(acl, src, src_size,
				       TABLE_SRC_COMPRESS, &src_order);
		if (IS_ERR(src_table))
			goto out;
	} else {
		set_up_addr(acl, virt_to_phys(src), PAGE_OFF(src), ADDR_SRC, 0);
	}

	if (_vmalloc_or_module_addr(dst)) {
		dst_table = create_map(acl, dst, dst_size, 1,
				       TABLE_DST_COMPRESS, &dst_order);
		if (IS_ERR(dst_table))
			goto out;
	} else if ((unsigned long)dst < TASK_SIZE) {
		dst_table = create_user_map(acl, dst, dst_size,
				       TABLE_DST_COMPRESS, &dst_order);
		if (IS_ERR(dst_table))
			goto out;
	} else {
		set_up_addr(acl, virt_to_phys(dst), PAGE_OFF(dst), ADDR_DST, 0);
	}
again:
	control        = 1;
	acl->algorithm = ALGORITHM_ZSTD;
	acl->end       = 1;
	acl->compress  = CMD_COMPRESS;
	acl->owner     = 1;
	acl->status    = 0xffffffff;
	if (!amfc->work_mode) {
		//acl->interrupt = 1;
		control |= (1 << 4);
	}

	amfc_map_addr((long)acl, sizeof(*acl), DMA_TO_DEVICE);
	amfc_hw_write(3, AMFC_GL_CMD0_IRQCLR);
	amfc_hw_write(virt_to_phys(acl) >> ADDR_SHIFT, AMFC_GL_CMD0_DESC_BASE_ADDR);
	/* trigger with irq en */
	amfc_hw_write(control, AMFC_GL_CMD0_CONTROL);

	if (!amfc->work_mode) {
		/* wait hw irq */
		ret = wait_for_completion_interruptible(&amfc->ccomp);
		if (ret) {
			ret = -EINTR;
			goto out;
		}
	} else {
		/* poll */
		tick = sched_clock();
		while (1) {
			status = amfc_hw_read(AMFC_GL_CMD0_STATUS);
			if ((status & IRQ_MASK))
				break;
			cur = sched_clock();
			if (cur - tick >= timeout * 1000) {
				pr_emerg_ratelimited("%s timeout:%lld -> %lld, %lld, tick:%d\n",
					 __func__, tick, cur, timeout, amfc_hw_read(AMFC_CMD0_TIME_MEASURE));
				show_regs(NULL);
				amfc_unmap_addr((long)acl, sizeof(*acl), DMA_FROM_DEVICE);
				show_acl(acl);
				pr_info_ratelimited("src:%px, dst:%px, src_size:%d, dst_size:%d\n", src, dst, (int)src_size, (int)dst_size);
				dump_addr(src, src_size);
				if (cur - tick >= timeout * 500UL)
					break;

				// init again and retry;
				amfc_hw_init();
				/* release for irq */
				local_irq_enable();
				local_irq_disable();
				goto again;
			}
			if (atomic_read(&amfc->in_suspend)) {
				ret = -ENODEV;
				goto out;
			}
		}
	}
	clks = amfc_hw_read(AMFC_CMD0_TIME_MEASURE);

	amfc_unmap_addr((long)acl, sizeof(*acl), DMA_FROM_DEVICE);
	status = amfc_hw_read(AMFC_GL_CMD0_STATUS);
	if (!(status & AMFC_ERROR_MASK))
		ret = acl->result_size;
	else
		ret = 0 - ((status & AMFC_ERROR_MASK) >> 8);
out:
	if (!IS_ERR_OR_NULL(src_table) && src_table != amfc->pages[TABLE_SRC_COMPRESS])
		__free_pages(src_table, src_order);
	if (!IS_ERR_OR_NULL(dst_table) && dst_table != amfc->pages[TABLE_DST_COMPRESS])
		__free_pages(dst_table, dst_order);
	if (atomic_read(&amfc->in_suspend)) /* check again */
		ret = -ENODEV;
	if (ret < 0) {
		if (ret == -ENODEV) {
			pr_err("%s compress failed caused by suspend\n", __func__);
		} else {
			amfc->in_enc_err = 1;
			if (((status & AMFC_ERR_MASK) >> 8) != AMFC_ENC_DST_SIZE_OVF) {
				pr_err_ratelimited("acl:%px, compress failed, src:%px, dst:%px, ssize:%d, dsize:%d, ret:%d, status:%x\n",
					acl, src, dst, (int)src_size, (int)dst_size,
					ret, amfc_hw_read(AMFC_GL_CMD0_STATUS));
				show_regs(NULL);
				show_acl(acl);
			}
			while (!spin_trylock(&amfc->dec_lock)) {
				if (amfc->in_dec_err) {
					amfc_hw_write(0x80000000, AMFC_GL_CMD0_CONTROL);
					goto error_handled;
				}
			}
			amfc_hw_write(0x80000000, AMFC_GL_CMD0_CONTROL);
			spin_unlock(&amfc->dec_lock);
error_handled:
			amfc->in_enc_err = 0;
		}
	} else {
		amfc->cin         += src_size;
		amfc->cout        += ret;
		amfc->total_ctick += clks;
		if (amfc->log > 2)
			pr_info("  compress acl:%px, src:%px, dst:%px, src size:%5d, result size:%5d, tick:%d\n",
				acl, src, dst, (int)src_size, ret, clks);
	}
	amfc_hw_write(0x03, AMFC_GL_CMD0_IRQCLR);
	if (amfc->work_mode == AMFC_POLL_MODE)
		spin_unlock(&amfc->com_lock);
	else
		spin_unlock_irqrestore(&amfc->com_lock, flags);

	if (ret > 0)
		amfc_unmap_addr((long)dst, ret > dst_size ? dst_size : ret, DMA_FROM_DEVICE);
	return ret;
}
EXPORT_SYMBOL(amfc_compress);

#ifndef CONFIG_AMLOGIC_LINUX_S_OPTIMIZE // for C4
/*
 * IRQ for compress
 */
static irqreturn_t amfc0_irq_handler(int irq, void *dev_instance)
{
	struct amfc *amfc = (struct amfc *)dev_instance;
	unsigned int val;

	val = amfc_hw_read(AMFC_GL_CMD0_STATUS);
	if (val & AMFC_IRQ_FLAG) { /* job done or error */
		complete(&amfc->ccomp);
	} else {
		pr_err("AMFC0 BUG: no valid IRQ flags but irq happen:%x\n", val);
	}
	amfc_hw_write(3, AMFC_GL_CMD0_IRQCLR); /* clear IRQ */

	return IRQ_HANDLED;
}

/*
 * IRQ for decompress
 */
static irqreturn_t amfc1_irq_handler(int irq, void *dev_instance)
{
	struct amfc *amfc = (struct amfc *)dev_instance;
	unsigned int val;

	val = amfc_hw_read(AMFC_GL_CMD1_STATUS);
	if (val & AMFC_IRQ_FLAG) { /* job done or error */
		complete(&amfc->dcomp);
	} else {
		pr_err("AMFC1 BUG: no valid IRQ flags but irq happen:%x\n", val);
	}
	amfc_hw_write(3, AMFC_GL_CMD1_IRQCLR); /* clear IRQ */

	return IRQ_HANDLED;
}

/* --------------------- crypto interface --------------------- */
// TODO: implement them
static int amfc_crypto_init(struct crypto_tfm *tfm)
{
	return 0;
}

static void amfc_crypto_exit(struct crypto_tfm *tfm)
{
}

static int amfc_crypto_compress(struct crypto_tfm *tfm, const u8 *src,
			 unsigned int slen, u8 *dst, unsigned int *dlen)
{
	int ret;

	ret = amfc_compress((void *)src, dst, slen, *dlen);
	if (ret < 0)
		return ret;
	*dlen = ret;
	return 0;
}

static int amfc_crypto_decompress(struct crypto_tfm *tfm, const u8 *src,
			   unsigned int slen, u8 *dst, unsigned int *dlen)
{
	int ret;

	ret = amfc_decompress((void *)src, dst, slen, *dlen, 0);
	if (ret < 0)
		return ret;
	*dlen = ret;
	return 0;
}

static struct crypto_alg amfc_alg = {
	.cra_name		= "amfc",
	.cra_driver_name	= "amfc-generic",
	.cra_flags		= CRYPTO_ALG_TYPE_COMPRESS,
	.cra_ctxsize		= 0,
	.cra_module		= THIS_MODULE,
	.cra_init		= amfc_crypto_init,
	.cra_exit		= amfc_crypto_exit,
	.cra_u			= {
		.compress = {
			.coa_compress	= amfc_crypto_compress,
			.coa_decompress	= amfc_crypto_decompress,
		}
	}
};

static int eamfc_crypto_decompress(struct crypto_tfm *tfm, const u8 *src,
			   unsigned int slen, u8 *dst, unsigned int *dlen)
{
	int ret;
	int off;
	unsigned int out_size;

	/* special handle for EROFS */
	// align src addr to  page header
	if (slen > PAGE_SIZE) {
		off = ((unsigned long)src) & ~PAGE_MASK;
		src -= off;
		slen += off;
	}
	out_size = *dlen;
	*dlen += AMFC_STREAM_MARGIN;
	ret = amfc_decompress((void *)src, dst, slen, *dlen, 1);
	if (ret < 0 && ((ret != -AMFC_DEC_DST_SIZE_OVF) && (ret != -AMFC_DEC_DST_PAGE_ERR)))
		return ret;
	*dlen = out_size;
	return 0;
}

/* for erofs*/
static struct crypto_alg eamfc_alg = {
	.cra_name		= "eamfc",
	.cra_driver_name	= "eamfc-generic",
	.cra_flags		= CRYPTO_ALG_TYPE_COMPRESS,
	.cra_ctxsize		= 0,
	.cra_module		= THIS_MODULE,
	.cra_init		= amfc_crypto_init,
	.cra_exit		= amfc_crypto_exit,
	.cra_u			= {
		.compress = {
			.coa_compress	= amfc_crypto_compress,
			.coa_decompress	= eamfc_crypto_decompress,
		}
	}
};

static int eamfc2_crypto_decompress(struct crypto_tfm *tfm, const u8 *src,
			   unsigned int slen, u8 *dst, unsigned int *dlen)
{
	int ret;
	unsigned int out_size;

	out_size = *dlen;
	*dlen += AMFC_STREAM_MARGIN;
	ret = amfc_decompress((void *)src, dst, slen, *dlen, 1);
	if (ret < 0 && ((ret != -AMFC_DEC_DST_SIZE_OVF) && (ret != -AMFC_DEC_DST_PAGE_ERR)))
		return ret;
	*dlen = out_size;
	return 0;
}

/* for erofs*/
static struct crypto_alg eamfc2_alg = {
	.cra_name		= "eamfc2",
	.cra_driver_name	= "eamfc2-generic",
	.cra_flags		= CRYPTO_ALG_TYPE_COMPRESS,
	.cra_ctxsize		= 0,
	.cra_module		= THIS_MODULE,
	.cra_init		= amfc_crypto_init,
	.cra_exit		= amfc_crypto_exit,
	.cra_u			= {
		.compress = {
			.coa_compress	= amfc_crypto_compress,
			.coa_decompress	= eamfc2_crypto_decompress,
		}
	}
};
#endif

/*----------- debug fs layer --------------*/
static ssize_t log_store(const struct class *cla,
			 const struct class_attribute *attr,
			 const char *buf, size_t count)
{
	unsigned long value = -1UL;

	if (kstrtoul(buf, 10, &value))
		return -EINVAL;
	amfc->log = value;
	return count;
}

static ssize_t log_show(const struct class *cla,
			const struct class_attribute *attr, char *buf)
{

	return sprintf(buf, "log:%d\n", amfc->log);
}
static CLASS_ATTR_RW(log);

static ssize_t statistics_store(const struct class *cla,
			        const struct class_attribute *attr,
			        const char *buf, size_t count)
{
	/* clear all statictics data */
	amfc->cin                = 0;
	amfc->cout               = 0;
	amfc->din                = 0;
	amfc->z_dout             = 0;
	amfc->e_dout             = 0;
	amfc->total_ctick        = 0;
	amfc->total_dtick        = 0;
	amfc->z_dtick            = 0;
	amfc->e_dtick            = 0;
	amfc->c_count            = 0;
	amfc->d_count            = 0;
	amfc->c_congestion       = 0;
	amfc->d_congestion       = 0;
#ifdef CONFIG_AMLOGIC_F2FS
	amfc->f2fs_enc_in        = 0;
	amfc->f2fs_enc_out       = 0;
	amfc->f2fs_dec_in        = 0;
	amfc->f2fs_dec_out       = 0;
	amfc->f2fs_enc_fail      = 0;
	amfc->f2fs_dec_fail      = 0;
#endif
	return count;
}

static ssize_t statistics_show(const struct class *cla,
			       const struct class_attribute *attr, char *buf)
{
	int s = 0;
	unsigned long long ctmp1, dtmp1, ctmp2, dtmp2;
	unsigned long rate = amfc->rate / 1000000;

	ctmp1 = amfc->cout * 100;
	do_div(ctmp1, amfc->cin);
	s += sprintf(buf + s, "Total compressed in:             %16lld\n", amfc->cin);
	s += sprintf(buf + s, "Total compressed out:            %16lld\n", amfc->cout);
	s += sprintf(buf + s, "Average compress ratio:          %16lld%%\n", ctmp1);
	s += sprintf(buf + s, "Total decompressed in:           %16lld\n", amfc->din);
	s += sprintf(buf + s, "Total decompressed out:          %16lld\n", amfc->z_dout + amfc->e_dout);

	ctmp1 = amfc->total_ctick;
	dtmp1 = amfc->total_dtick;
	do_div(ctmp1, rate);
	do_div(dtmp1, rate);
	s += sprintf(buf + s, "Total compressed tick(us):       %16lld\n", ctmp1);
	s += sprintf(buf + s, "Total decompressed tick(us):     %16lld\n", dtmp1);

	ctmp2 = amfc->cin;
	dtmp2 = amfc->z_dout + amfc->e_dout;
	do_div(ctmp2, ctmp1);
	do_div(dtmp2, dtmp1);
	s += sprintf(buf + s, "Average compress speed(MB/s):    %16lld\n", ctmp2);
	s += sprintf(buf + s, "Average decompress speed(MB/s):  %16lld\n", dtmp2);

	dtmp1 = amfc->z_dtick;
	do_div(dtmp1, rate);
	dtmp2 = amfc->z_dout;
	do_div(dtmp2, dtmp1);
	s += sprintf(buf + s, "Avg ZRAM decompress speed(MB/s): %16lld\n", dtmp2);

	dtmp1 = amfc->e_dtick;
	do_div(dtmp1, rate);
	dtmp2 = amfc->e_dout;
	do_div(dtmp2, dtmp1);
	s += sprintf(buf + s, "Avg EROFS decompress speed(MB/s):%16lld\n", dtmp2);

	s += sprintf(buf + s, "Clock speed(Hz):                 %16ld\n",  amfc->rate);
	s += sprintf(buf + s, "Compressed count:                %16ld\n",  amfc->c_count);
	s += sprintf(buf + s, "Decompressed count:              %16ld\n",  amfc->d_count);
	s += sprintf(buf + s, "Compress congestion count:       %16ld\n",  amfc->c_congestion);
	s += sprintf(buf + s, "Decompress congestion count:     %16ld\n",  amfc->d_congestion);
#ifdef CONFIG_AMLOGIC_F2FS
	s += sprintf(buf + s, "f2fs_enc_in:                     %16lld\n",  amfc->f2fs_enc_in);
	s += sprintf(buf + s, "f2fs_enc_out:                    %16lld\n",  amfc->f2fs_enc_out);
	s += sprintf(buf + s, "f2fs_dec_in:                     %16lld\n",  amfc->f2fs_dec_in);
	s += sprintf(buf + s, "f2fs_dec_out:                    %16lld\n",  amfc->f2fs_dec_out);
	s += sprintf(buf + s, "f2fs_enc_fail:                   %16ld\n",  amfc->f2fs_enc_fail);
	s += sprintf(buf + s, "f2fs_dec_fail:                   %16ld\n",  amfc->f2fs_dec_fail);
#endif

	return s;
}
static CLASS_ATTR_RW(statistics);

static ssize_t work_mode_store(const struct class *cla,
			       const struct class_attribute *attr,
			       const char *buf, size_t count)
{
	unsigned long value = -1UL;

	if (kstrtoul(buf, 16, &value))
		return -EINVAL;
	if (value >= 3) {
		pr_info("invalid input:%s\n", buf);
		return -EINVAL;
	}
	amfc->work_mode = value;
	return count;
}

static ssize_t work_mode_show(const struct class *cla,
			      const struct class_attribute *attr, char *buf)
{
	return sprintf(buf, "work_mode:%d\n", amfc->work_mode);
}
static CLASS_ATTR_RW(work_mode);

static ssize_t clk_store(const struct class *cla,
			 const struct class_attribute *attr,
			 const char *buf, size_t count)
{
	unsigned long value = -1UL;
	int i;

	if (kstrtoul(buf, 10, &value))
		return -EINVAL;
	for (i = 0; i < ARRAY_SIZE(amfc_clk); i++) {
		if (amfc_clk[i] == value)
			break;
	}
	if (i >= ARRAY_SIZE(amfc_clk)) {
		pr_emerg("unsupported clk:%ld\n", value);
		return -1;
	}
	clk_set_rate(amfc->clk, value);
	amfc->rate = value;
	return count;
}

static ssize_t clk_show(const struct class *cla,
			const struct class_attribute *attr, char *buf)
{
	return sprintf(buf, "FREQ:%ld hz\n", clk_get_rate(amfc->clk));
}
static CLASS_ATTR_RW(clk);

static ssize_t reg_dump_store(const struct class *cla,
			      const struct class_attribute *attr,
			      const char *buf, size_t count)
{
	unsigned int off, value;

	if (sscanf(buf, "%x %x\n", &off, &value) != 2)
		return -EINVAL;

	if (off > AMFC_MIF_QOS_UGT)
		return -EINVAL;

	amfc_hw_write(value, off);
	return count;
}

static ssize_t reg_dump_show(const struct class *cla,
			     const struct class_attribute *attr, char *buf)
{
	int size;

	size = show_regs(buf);
	return size;
}
static CLASS_ATTR_RW(reg_dump);

#ifdef CONFIG_AMFC_DMA_TEST
static unsigned int test_should_stop;
static int test_size, test_count, test_delay;
/*
 * src: source buffer that need decompress
 * dst: destination buffer that need store
 * src_size: size of source buffer, dst buffer must have same or larger size
 *           compared with src buffer
 * return value: 0 for success, negative value if failed
 */
static int amfc_copy_bandwidth_test(void *src, void *dst, ssize_t src_size)
{
	struct page *src_table = NULL, *dst_table = NULL;
	struct amfc_cmd_list *acl;
	int ret = -EINVAL;
	int src_order = 1, dst_order = 1;		// force allocate page table
	unsigned int status, control, clks = 0;
	unsigned long flags;
	unsigned long long timeout = ((src_size) / PAGE_SIZE) * 50 + 5000; // us
	unsigned long long tick, cur;

	if (!amfc)
		return -ENOMEM;

	if (!src || !dst || !src_size)
		return -EINVAL;

	acl = kmalloc(CACHELINE_SIZE, GFP_KERNEL);
	if (!acl)
		return -ENOMEM;

	amfc_map_addr((long)dst, src_size, DMA_FROM_DEVICE);
	amfc_map_addr((long)src, src_size, DMA_TO_DEVICE);

	amfc->d_count++;

	/* setup command list, decompress use cmd0 */
	memset(acl, 0, sizeof(struct amfc_cmd_list));
	acl->src_size = src_size;
	acl->dst_size = src_size;
	if (_vmalloc_or_module_addr(src)) {
		src_table = create_map(acl, src, src_size, 0,
				       TABLE_SRC_DECOMPRESS, &src_order);
		if (IS_ERR(src_table))
			goto out;
	} else if ((unsigned long)src < TASK_SIZE) {
		src_table = create_user_map(acl, src, src_size,
				       TABLE_SRC_DECOMPRESS, &src_order);
		if (IS_ERR(src_table))
			goto out;
	} else {
		set_up_addr(acl, virt_to_phys(src), PAGE_OFF(src), ADDR_SRC, 0);
	}

	if (_vmalloc_or_module_addr(dst)) {
		dst_table = create_map(acl, dst, src_size, 0,
				       TABLE_DST_DECOMPRESS, &dst_order);
		if (IS_ERR(dst_table))
			goto out;
	} else if ((unsigned long)dst < TASK_SIZE) {
		dst_table = create_user_map(acl, dst, src_size,
				       TABLE_DST_DECOMPRESS, &dst_order);
		if (IS_ERR(dst_table))
			goto out;
	} else {
		set_up_addr(acl, virt_to_phys(dst), PAGE_OFF(dst), ADDR_DST, 0);
	}

	control        = 1;
	acl->algorithm = ALGORITHM_DMA_COPY;
	acl->end       = 1;
	acl->compress  = CMD_DECOMPRESS;
	if (!amfc->work_mode)
		control |= (1 << 4);

	do {
		acl->owner     = 1;
		acl->status    = 0xffffffff;
		amfc_map_addr((long)acl, sizeof(*acl), DMA_TO_DEVICE);
		if (test_delay > 0)
			usleep_range(test_delay, test_delay + 1);
		spin_lock_irqsave(&amfc->dec_lock, flags);

		amfc_hw_write(3, AMFC_GL_CMD1_IRQCLR);
		amfc_hw_write(virt_to_phys(acl) >> ADDR_SHIFT, AMFC_GL_CMD1_DESC_BASE_ADDR);
		/* trigger with irq en */
		amfc_hw_write(control, AMFC_GL_CMD1_CONTROL);

		/* poll */
		tick = sched_clock();
		while (1) {
			status = amfc_hw_read(AMFC_GL_CMD1_STATUS);
			if ((status & IRQ_MASK))
				break;
			cur = sched_clock();
			if (cur - tick >= timeout * 1000) {
				pr_emerg("%s timeout:%lld -> %lld, %lld\n",
					 __func__, tick, cur, timeout);
				show_regs(NULL);
				amfc_unmap_addr((long)acl, sizeof(*acl), DMA_FROM_DEVICE);
				show_acl(acl);
				if (cur - tick >= timeout * 50000UL)
					break;
				// init again and retry;
				amfc_hw_init();
			}
		}

		amfc_unmap_addr((long)acl, sizeof(*acl), DMA_FROM_DEVICE);
		status = amfc_hw_read(AMFC_GL_CMD1_STATUS);
		amfc_hw_write(0x03, AMFC_GL_CMD1_IRQCLR);
		spin_unlock_irqrestore(&amfc->dec_lock, flags);

		if (test_count < 0 && test_should_stop)
			break;

		test_count--;
	} while (test_count);

	if (!(status & AMFC_ERROR_MASK))
		ret = acl->result_size;
	else
		ret = 0 - ((status & AMFC_ERROR_MASK) >> 8);
out:
	if (!IS_ERR_OR_NULL(src_table) && src_table != amfc->pages[TABLE_SRC_DECOMPRESS])
		__free_pages(src_table, src_order);
	if (!IS_ERR_OR_NULL(dst_table) && dst_table != amfc->pages[TABLE_DST_DECOMPRESS])
		__free_pages(dst_table, dst_order);
	if (atomic_read(&amfc->in_suspend)) /* check again */
		ret = -ENODEV;
	if (ret < 0) {
		if (ret == -ENODEV) {
			pr_err("%s, copy failed caused by suspend\n", __func__);
		} else {
			amfc->in_dec_err = 1;
			pr_err("acl:%px, copy failed, src:%px, dst:%px, ssize:%d, ret:%d, status:%x\n",
				acl, src, dst, (int)src_size,
				ret, amfc_hw_read(AMFC_GL_CMD1_STATUS));
			/* sw reset */
			while (!spin_trylock(&amfc->com_lock)) {
				if (amfc->in_enc_err) {
					amfc_hw_write(0x80000000, AMFC_GL_CMD1_CONTROL);
					goto error_handled;
				}
			}
			amfc_hw_write(0x80000000, AMFC_GL_CMD1_CONTROL);
			spin_unlock(&amfc->com_lock);
error_handled:
			amfc->in_dec_err = 0;
		}
	} else {
		if (amfc->log)
			pr_info("copy ACL:%px, src:%px, dst:%px, src size:%5d, result size:%5d, tick:%d\n",
				acl, src, dst, (int)src_size, ret, clks);
	}
	amfc_unmap_addr((long)dst, src_size, DMA_FROM_DEVICE);

	kfree(acl);
	test_should_stop = 0;

	return ret;
}

static int amfc_band_test_work(void *data)
{
	void *src, *dst;
	int ret;

	src = vmalloc(test_size * SZ_1K);
	dst = vmalloc(test_size * SZ_1K);
	if (!src || !dst) {
		pr_info("%s, alloc %d kb failed, src:%px, dst:%px\n",
			__func__, test_size, src, dst);
		if (src)
			vfree(src);
		if (dst)
			vfree(dst);
		return -ENOMEM;
	}
	ret = amfc_copy_bandwidth_test(src, dst, test_size * SZ_1K);
	vfree(src);
	vfree(dst);
	pr_info("%s test finished, ret:%d\n", __func__, ret);

	return 0;
}

static ssize_t band_test_store(const struct class *cla,
			      const struct class_attribute *attr,
			      const char *buf, size_t count)
{
	int size, c = 0, delay = 0;

	test_should_stop = 0;
	if (strncmp(buf, "stop", 4) == 0) {
		test_should_stop = 1;
		return count;
	}

	if (strncmp(buf, "default", 7) == 0) {
		size = 4096;
		c = -1;
		delay = 3043;
		goto start_test;
	}

	if (sscanf(buf, "%d %d %d\n", &size, &c, &delay) != 3) {
		pr_err("Must be format: [test_size] [test_count] [test_delay]\n");
		pr_err("test_size: in Kb\n");
		pr_err("test_delay: in us\n");
		return -EINVAL;
	}

	if (size < 0) {
		pr_err("Invalid test size:%d\n", size);
		return -EINVAL;
	}

start_test:
	test_size  = size;
	test_count = c;
	test_delay = delay;
	pr_info("%s, test_size:%d kb, count:%d delay:%d us\n", __func__, size, c, delay);
	kthread_run(amfc_band_test_work, NULL, "amfc_band_test");

	return count;
}

static CLASS_ATTR_WO(band_test);
#endif

#ifdef CONFIG_AMFC_TEST
static char test_buf[PAGE_SIZE * 4] __aligned(4096);
static ssize_t zram_test_store(const struct class *cla,
			       const struct class_attribute *attr,
			       const char *buf, size_t count)
{
	struct lruvec *lruvec;
	pg_data_t *pgdat, *tmp;
	struct page *page, *next;
	struct list_head *list;
	void *src;
	int ret, lru;
	struct mem_cgroup *root = NULL, *memcg;
	int pages = 0;

	kstrtoint(buf, 10, &pages);
	if (pages <= 0)
		return count;
	for_each_online_pgdat(pgdat) {
		memcg = mem_cgroup_iter(root, NULL, NULL);
		do {
			lruvec = mem_cgroup_lruvec(memcg, pgdat);
			tmp = lruvec_pgdat(lruvec);

			for_each_lru(lru) {
				/* only count for filecache */
				if (is_file_lru(lru) &&
				    lru != LRU_UNEVICTABLE)
					continue;

				list = &lruvec->lists[lru];
				//spin_lock(&tmp->lru_lock);
				list_for_each_entry_safe(page, next,
							 list, lru) {
					src = kmap(page);
					ret = amfc_compress(src, test_buf,
							    PAGE_SIZE, PAGE_SIZE);
					if (ret > 0)
						ret = amfc_decompress(test_buf,
								      test_buf + PAGE_SIZE,
								      ret, PAGE_SIZE);
					kunmap(page);
					pages--;
					if (pages <= 0)
						return count;
				}
				//spin_unlock(&tmp->lru_lock);
			}
		} while ((memcg =  mem_cgroup_iter(root, memcg, NULL)));
	}

	return count;
}
static CLASS_ATTR_WO(zram_test);

static ssize_t zfile_test_store(const struct class *cla,
				const struct class_attribute *attr,
				const char *buf, size_t count)
{
	int i = 0, size, ret;
	unsigned long addr;
	void *p;

	if (sscanf(buf, "%lx %x", &addr, &size) != 2)
		return -EINVAL;

	if (size < 0)
		return count;

	pr_info("%s, input addr:%lx, size:%x\n", __func__, addr, size);
	p = ioremap(addr, size);
	if (!p)
		return count;
	while (i < size) {
		ret = amfc_compress(p + i, test_buf, PAGE_SIZE, PAGE_SIZE);
		if (ret > 0)
			amfc_decompress(test_buf, test_buf + 2 * PAGE_SIZE, ret, PAGE_SIZE);
		i += PAGE_SIZE;
	}
	iounmap(p);

	return count;
}
static CLASS_ATTR_WO(zfile_test);
#endif

static struct attribute *amfc_attrs[] = {
	&class_attr_log.attr,
	&class_attr_reg_dump.attr,
	&class_attr_work_mode.attr,
	&class_attr_statistics.attr,
	&class_attr_clk.attr,
#ifdef CONFIG_AMFC_DMA_TEST
	&class_attr_band_test.attr,
#endif
#ifdef CONFIG_AMFC_TEST
	&class_attr_zram_test.attr,
	&class_attr_zfile_test.attr,
#endif
	NULL
};
ATTRIBUTE_GROUPS(amfc);

static struct class amfc_class = {
	.name = "amfc",
	.class_groups = amfc_groups,
};

static int amfc_open(struct inode *inode, struct file *filp)
{
	return 0;
}

int amfc_close(struct inode *inode, struct file *filp)
{
	return 0;
}

long amfc_ioctrl(struct file *filp, unsigned int cmd, unsigned long userdata)
{
	struct amfc_args arg;
	struct amfc_args __user *argp;
	int ret = -2;

	if (_IOC_TYPE(cmd) != AMFC_IOC_MAGIC)
		return -EINVAL;

	argp = (void __user *)userdata;
	if (copy_from_user((void *)&arg, argp, sizeof(arg)))
		return -ENOMEM;

	pr_debug("user arg, src:%llx, dst:%llx, src_size:%d, dst_size:%d\n",
		arg.src_addr, arg.dst_addr, arg.src_size, arg.dst_size);
	if (!arg.src_addr || !arg.dst_addr || !arg.src_size || !arg.dst_size) {
		pr_debug("NULL addr or size\n");
		return -EINVAL;
	}
	if (arg.src_size > AMFC_MAX_FRAME_SIZE || arg.dst_size > AMFC_MAX_FRAME_SIZE) {
		pr_debug("Input size over flow\n");
		return -ERANGE;
	}

	/* input source and dest address must be page aligned */
	if ((arg.src_addr & ~PAGE_MASK) || (arg.dst_addr & ~PAGE_MASK)) {
		pr_debug("address not 4KB aligned\n");
		return -EFAULT;
	}

	/* should be mapped already */
	if (!access_ok((void *)((long)arg.src_addr), arg.src_size)) {
		pr_debug("src can't full access\n");
		return -EFAULT;
	}

	if (!access_ok((void *)((long)arg.dst_addr), arg.dst_size)) {
		pr_debug("dst can't full access\n");
		return -EFAULT;
	}

	pr_debug("user arg check pass\n");
	switch (cmd) {
	case AMFC_IOC_COMPRESS:
		ret = amfc_compress((void *)((long)arg.src_addr),
				    (void *)((long)arg.dst_addr),
				    arg.src_size, arg.dst_size);
		break;
	case AMFC_IOC_DECOMPRESS:
		ret = amfc_decompress((void *)((long)arg.src_addr),
				      (void *)((long)arg.dst_addr),
				      arg.src_size, arg.dst_size, 0);
		break;
	default:
		return -EINVAL;
	}

	return ret;
}

#ifdef CONFIG_COMPAT
static long amfc_compat_ioctl(struct file *file, unsigned int cmd, ulong arg)
{
	long ret = 0;

	ret = amfc_ioctrl(file, cmd, (ulong)compat_ptr(arg));
	return ret;
}
#endif

const struct file_operations amfc_fops = {
	.owner          = THIS_MODULE,
	.open           = amfc_open,
	.release        = amfc_close,
	.unlocked_ioctl = amfc_ioctrl,
#ifdef CONFIG_COMPAT
	.compat_ioctl   = amfc_compat_ioctl,
#endif
};

/*------------ driver layer ---------------*/
static void amfc_remove(struct platform_device *pdev)
{
	int i;
	struct amfc *tmp = amfc;

	if (amfc) {
		amfc = NULL;
		device_destroy(&amfc_class, MKDEV(MAJOR(amfc_devno), 0));
		class_destroy(&amfc_class);
		for (i = 0; i < 4; i++) {
			if (tmp->pages[i])
				__free_pages(tmp->pages[i], 0);
		}
		vfree(tmp->bounce_buffer);
		kfree(tmp->compress);
		kfree(tmp->decompress);
		devm_kfree(&pdev->dev, tmp);
		if (amfc_cdev)
			cdev_del(amfc_cdev);
		if (amfc_devno)
			unregister_chrdev_region(amfc_devno, 1);
		kfree(amfc_cdev);
	}
}

void amfc_syscore_shutdown(void)
{
#if 0	// need always on
	struct platform_device *pdev;

	pdev = to_platform_device(amfc->dev);
	amfc_remove(pdev);
	pm_runtime_force_suspend(&pdev->dev);
#endif
}

static struct syscore_ops amfc_syscore_ops = {
	.shutdown = amfc_syscore_shutdown,
};

/* vendor hook to probe compression of f2fs */
static int amfc_probe(struct platform_device *pdev)
{
	int r = 0, num;
	unsigned int tmp;
	struct resource *res;
#ifndef CONFIG_AMLOGIC_LINUX_S_OPTIMIZE
	int irq;
#endif
	struct device_node *node;
	struct device *dev;

#ifdef CONFIG_AMLOGIC_F2FS
	r = hook_f2fs();
	if (r == -ENOMEM)
		return r;
	else if (r)
		pr_emerg("f2fs may enabled compress but amfc hook failed\n");
#endif

	amfc = devm_kzalloc(&pdev->dev, sizeof(*amfc), GFP_KERNEL);
	if (!amfc)
		return -ENOMEM;

	garbage_page = alloc_pages(GFP_KERNEL, 0);
	if (!garbage_page)
		return -ENOMEM;

	node = pdev->dev.of_node;
	amfc->dev = &pdev->dev;

	dma_set_mask(amfc->dev, DMA_BIT_MASK(36));
	amfc->compress = kzalloc(sizeof(*amfc->compress) + 1, GFP_KERNEL);
	amfc->decompress = kzalloc(sizeof(*amfc->decompress) + 1, GFP_KERNEL);
	BUG_ON((unsigned long)amfc->compress & (CACHELINE_SIZE - 1));
	BUG_ON((unsigned long)amfc->decompress & (CACHELINE_SIZE - 1));
	if (!amfc->compress || !amfc->decompress) {
		r = -ENOMEM;
		goto err;
	}

	for (num = 0; num < 4; num++) {
		amfc->pages[num] = alloc_pages(GFP_KERNEL, 0);
		if (!amfc->pages[num]) {
			r = -ENOMEM;
			goto err;
		}
	}
	tmp = (unsigned long)of_device_get_match_data(&pdev->dev);
	pr_info("%s, socs:%d, coherent:%d, garbage page:%lx, bounce:%px, c:%px, d:%px\n",
		__func__, tmp, dev_is_dma_coherent(amfc->dev),
		page_to_pfn(garbage_page), amfc->bounce_buffer, amfc->compress, amfc->decompress);
	amfc->chip = tmp;

	if (!amfc_supported())
		return -ENODEV;

	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	if (res)
		amfc->io_base = ioremap(res->start, resource_size(res));
	if (!amfc->io_base) {
		pr_err("map io base failed:%px\n", res);
		r = -EIO;
		goto err;
	}
	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
	if (res)
		amfc->clk_base = ioremap(res->start, resource_size(res));
	if (!amfc->clk_base) {
		pr_err("map clk base failed:%px\n", res);
		r = -EIO;
		goto err;
	}

	/*
	 * Test on T6D:
	 * ZRAM  average compress time:6.7us, max:16us, decompress is faster
	 * than compress of ZRAM. EROFS decomperss average time:18us, max:69us.
	 * There are too many atomic envs in upper layer caller,so can't
	 * use schedule, if with IRQ enabled when spinlock, AMFC hardware
	 * may be interrupted by other long IRQ over 7 ~ 9 ms,
	 * this will cause long waiting time of AMFC upper caller compared
	 * with IRQ disabled. So finally chose spinlock with IRQ off.
	 */
	amfc->work_mode = AMFC_POLL_IRQ_OFF;
#ifndef CONFIG_AMLOGIC_LINUX_S_OPTIMIZE
	if (!amfc->work_mode) {
		irq = platform_get_irq_byname(pdev, "amfc0");
		r = request_irq(irq, amfc0_irq_handler,
				IRQF_SHARED, "amfc0", amfc);
		if (r < 0)
			pr_err("request irq failed:%d, r:%d\n", irq, r);

		irq = platform_get_irq_byname(pdev, "amfc1");
		r = request_irq(irq, amfc1_irq_handler,
				IRQF_SHARED, "amfc1", amfc);
		if (r < 0)
			pr_err("request irq failed:%d, r:%d\n", irq, r);
	}
#endif
	amfc->clk = devm_clk_get(&pdev->dev, NULL);
	if (!amfc->clk) {
		pr_err("%s, can't get clk\n", __func__);
		r = -EPROBE_DEFER;
		goto err;
	}
	clk_prepare_enable(amfc->clk);
	amfc->rate = clk_get_rate(amfc->clk);
	pm_runtime_enable(amfc->dev);
	r = pm_runtime_get_sync(amfc->dev);
	if (r) {
		pr_err("get runtime pm failed:%d\n", r);
		r = -EPROBE_DEFER;
		goto err;
	}

	fix_amfc_clock(pdev);
	init_completion(&amfc->ccomp);
	init_completion(&amfc->dcomp);
	spin_lock_init(&amfc->com_lock);
	spin_lock_init(&amfc->dec_lock);
	atomic_set(&amfc->in_suspend, 0);
	amfc_hw_init();

	amfc->log = default_log;

	if (amfc->hw_version < AMFC_VER_1_1) {
		amfc->bounce_buffer = vzalloc(128 * 1024 + AMFC_STREAM_MARGIN);
		if (!amfc->bounce_buffer)
			goto err;
	}

#ifndef CONFIG_AMLOGIC_LINUX_S_OPTIMIZE
	r = crypto_register_alg(&amfc_alg);
	if (r) {
		pr_err("register amfc crypto failed:%d\n", r);
		goto err;
	}

	r = crypto_register_alg(&eamfc_alg);
	if (amfc->hw_version > AMFC_VER_1_0)
		r = crypto_register_alg(&eamfc2_alg);
	if (r) {
		crypto_unregister_alg(&amfc_alg);
		pr_err("register eamfc crypto failed:%d\n", r);
		goto err;
	}
#endif

	r = class_register(&amfc_class);
	if (r)
		pr_info("%s, class regist failed:%d\n", __func__, r);

	r = alloc_chrdev_region(&amfc_devno, 0, 1, "amfc");
	if (r < 0) {
		pr_emerg("alloc amfc cdev no failed\n");
		goto err;
	}
	pr_info("%s, dev no:%d\n", __func__, amfc_devno);
	amfc_cdev = kmalloc(sizeof(*amfc_cdev), GFP_KERNEL);
	if (!amfc_cdev) {
		r = -ENOMEM;
		pr_emerg("alloc amfc cdev failed\n");
		goto err;
	}
	cdev_init(amfc_cdev, &amfc_fops);
	amfc_cdev->owner = THIS_MODULE;
	r = cdev_add(amfc_cdev, amfc_devno, 1);
	if (r) {
		pr_emerg("amfc cdev add failed\n");
		goto err;
	}

	dev = device_create(&amfc_class, NULL, MKDEV(MAJOR(amfc_devno), 0), NULL, "amfc");
	if (IS_ERR_OR_NULL(dev)) {
		pr_emerg("amfc cdev create failed:%px\n", dev);
		r = -EINVAL;
		goto err;
	}
	register_syscore_ops(&amfc_syscore_ops);
	amfc->init_done = 1;
	pr_info("%s,%d amfc probe done\n", __func__, __LINE__);
	return 0;
err:
	for (num = 0; num < 4; num++) {
		if (amfc->pages[num])
			__free_pages(amfc->pages[num], 0);
	}
	if (amfc_cdev)
		cdev_del(amfc_cdev);
	if (amfc_devno)
		unregister_chrdev_region(amfc_devno, 1);
	kfree(amfc_cdev);
	kfree(amfc->compress);
	kfree(amfc->decompress);
	devm_kfree(&pdev->dev, amfc);
	amfc = NULL;
	return r;
}

static int __maybe_unused amfc_suspend(struct device *dev)
{
	atomic_inc(&amfc->in_suspend);
	clk_disable_unprepare(amfc->clk);
	return 0;
}

static int __maybe_unused amfc_resume(struct device *dev)
{
	clk_prepare_enable(amfc->clk);
	amfc_hw_init();
	atomic_dec(&amfc->in_suspend);
	return 0;
}

static SIMPLE_DEV_PM_OPS(amfc_pm_ops, amfc_suspend, amfc_resume);

#ifdef CONFIG_OF
static const struct of_device_id amfc_match[] = {
	{
		.compatible = "amlogic,amfc-s7d",
		.data = (void *)AMFC_S7D,
	},
	{
		.compatible = "amlogic,amfc-t6d",
		.data = (void *)AMFC_T6D,
	},
	{
		.compatible = "amlogic,amfc-t6x",
		.data = (void *)AMFC_T6X,
	},
	{
		.compatible = "amlogic,amfc-c4",
		.data = (void *)AMFC_C4,
	},
	{}
};
#endif

static struct platform_driver amfc_driver = {
	.driver = {
		.name  = "amfc",
		.owner = THIS_MODULE,
		.pm = &amfc_pm_ops,
		.of_match_table = amfc_match,
	},
	.remove = amfc_remove,
	.probe  = amfc_probe,
};

int __init amfc_init(void)
{
	int ret;

	ret = platform_driver_register(&amfc_driver);
	return ret;
}

void __exit amfc_exit(void)
{
	platform_driver_unregister(&amfc_driver);
}

#if IS_ENABLED(CONFIG_AMLOGIC_FREERTOS_MEMORY_FREE) && IS_BUILTIN(CONFIG_AMLOGIC_AMFC)
subsys_initcall_sync(amfc_init);
#else
module_init(amfc_init);
#endif
module_exit(amfc_exit);
MODULE_LICENSE("GPL");