// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
/*
 * Copyright (c) 2019 Amlogic, Inc. All rights reserved.
 */
#undef pr_fmt
#define pr_fmt(fmt) "memtest: " fmt

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/kthread.h>
#include <linux/delay.h>
#include <linux/mm.h>
#include <linux/vmalloc.h>
#include <linux/sched/clock.h>
#include <linux/platform_device.h>
#include <linux/dma-mapping.h>
#include <asm/cacheflush.h>
#include <linux/parser.h>
#include <linux/ctype.h>
#include <linux/string.h>
#include <linux/random.h>
#include <linux/amlogic/gki_module.h>

#define MODE_COPY 0
#define MODE_RDONLY 1
#define MODE_WRONLY 2
#define MODE_RANDOM 3
#define MODE_MAX 4

#define MEM_TYPE_NORMAL 0
#define MEM_TYPE_NONCACHED 1
#define MEM_TYPE_WRITECOMBINE 2
#define MEM_TYPE_MAX 3

#define SUBLOOPS 100
#define MAX_SUBLOOPS 100000

struct memtest_option {
	int mode;
	int mem_type;
	int size_kb;
	int cpumask;
	int min_pfn;
	int max_pfn;
	int loops;
	int subloops;
	int loop_sleepms;
	int disable_irq;
	int rand_cnt;
	int print;
};

enum {
	OPT_MODE,
	OPT_MEM_TYPE,
	OPT_SIZE_KB,
	OPT_CPUMASK,
	OPT_MIN_PFN,
	OPT_MAX_PFN,
	OPT_LOOPS,
	OPT_SUBLOOPS,
	OPT_LOOP_SLEEPMS,
	OPT_DISABLE_IRQ,
	OPT_RAND_CNT,
	OPT_PRINT,
	OPT_STOP,
	OPT_ERR,
};

static int memtest_stop;
//threads created by do_memtest but not yet exited; stop must wait
//for this to drain too, or a late-starting thread could register
//after stop already saw an empty task list and run forever
static atomic_t memtest_pending = ATOMIC_INIT(0);
static DEFINE_MUTEX(memtest_lock);
static LIST_HEAD(memtest_tasks);
static DECLARE_WAIT_QUEUE_HEAD(memtest_waitq);

struct memtest_task {
	struct task_struct *task;
	struct list_head list;
};

static void do_memtest_stop(void)
{
	pr_emerg("memtest: stopping all threads\n");

	//wake threads sleeping between loops so they see the stop
	//flag immediately
	wake_up(&memtest_waitq);

	//wait for the active task list to drain; each thread removes
	//itself and wakes us up on exit. memtest_pending covers threads
	//that were created but have not run/registered yet
	wait_event(memtest_waitq, list_empty(&memtest_tasks) &&
				  !atomic_read(&memtest_pending));

	pr_emerg("memtest: all threads stopped\n");
	//reset so new tests can be triggered afterwards
	memtest_stop = 0;
}

static char *mode_str[MODE_MAX] = {"copy", "rdonly", "wronly", "random"};
static char *mem_type_str[MEM_TYPE_MAX] = {"normal", "noncached", "writecombine"};

static const match_table_t memtest_tokens = {
	{OPT_MODE, "rw=%s"},
	{OPT_MEM_TYPE, "cache=%s"},
	{OPT_SIZE_KB, "size_kb=%d"},
	{OPT_CPUMASK, "cpumask=%x"},
	{OPT_MIN_PFN, "min_pfn=%x"},
	{OPT_MAX_PFN, "max_pfn=%x"},
	{OPT_LOOPS, "loops=%d"},
	{OPT_SUBLOOPS, "subloops=%d"},
	{OPT_LOOP_SLEEPMS, "loop_sleepms=%d"},
	{OPT_DISABLE_IRQ, "disable_irq=%d"},
	{OPT_RAND_CNT, "rand_cnt=%d"},
	{OPT_PRINT, "print=%d"},
	{OPT_STOP, "stop"},
	{OPT_ERR, NULL},
};

static void do_memtest(struct memtest_option *option);

static void show_help(char *err)
{
	if (err)
		pr_emerg("Error: %s\n", err);

	pr_emerg("Function:\n");
	pr_emerg(" - trigger a memtest thread or stop all memtest threads\n");
	pr_emerg("Usage:\n");
	pr_emerg(" - echo [option1=value1] [option2=value2] [...] >  memtest_trigger\n");
	pr_emerg("Option list:\n");
	pr_emerg("  - rw=copy|rdonly|wronly|random             (default copy)\n");
	pr_emerg("  - cache=normal|noncached|writecombine      (default normal)\n");
	pr_emerg("  - size_kb=alloc_memory_size_kb             (default 65536)\n");
	pr_emerg("  - cpumask=thread_cpu_mask                  (default 0xff)\n");
	pr_emerg("  - min_pfn=alloc_memory_min_pfn             (default 0x0)\n");
	pr_emerg("  - max_pfn=alloc_memory_max_pfn             (default 0xffffffff)\n");
	pr_emerg("  - loops=test_loops_num                     (default -1, means dead loop)\n");
	pr_emerg("  - subloops=test_subloops_num_per_loop      (default 100, max 100000)\n");
	pr_emerg("  - loop_sleepms=sleepsm_between_each_loop   (default 0, don't sleep)\n");
	pr_emerg("  - disable_irq=1|0                          (default 1, disable irq when test)\n");
	pr_emerg("  - rand_cnt=rand_access_cnt_per_subloop     (default 1000, means 1000 * 100)\n");
	pr_emerg("  - print=1|0                                (default 1, print test result)\n");
	pr_emerg("  - stop                                     (only stop all threads)\n");
	pr_emerg("\n");
}

#define NUMBER_BUF_LEN 24
static size_t aml_match_strlcpy(char *dest, const substring_t *src, size_t size)
{
	size_t ret = src->to - src->from;

	if (size) {
		size_t len = ret >= size ? size - 1 : ret;

		memcpy(dest, src->from, len);
		dest[len] = '\0';
	}
	return ret;
}

static int match_number(substring_t *s, int *result, int base)
{
	char *endp;
	char buf[NUMBER_BUF_LEN];
	int ret;
	long val;

	if (aml_match_strlcpy(buf, s, NUMBER_BUF_LEN) >= NUMBER_BUF_LEN)
		return -ERANGE;

	ret = 0;
	val = simple_strtol(buf, &endp, base);
	if (endp == buf)
		ret = -EINVAL;
	else if (val < (long)INT_MIN || val > (long)INT_MAX)
		ret = -ERANGE;
	else
		*result = (int)val;

	return ret;
}

//match_hex() is unavabile in gki20
static int aml_match_hex(substring_t *s, int *result)
{
	return match_number(s, result, 16);
}

//aml_match_token()/aml_match_int()/aml_match_strdup() are unavailable in gki,
//copied from lib/parser.c and renamed with aml_ prefix
static int aml_match_one(char *s, const char *p, substring_t args[])
{
	char *meta;
	int argc = 0;

	if (!p)
		return 1;

	while (1) {
		int len = -1;
		meta = strchr(p, '%');
		if (!meta)
			return strcmp(p, s) == 0;

		if (strncmp(p, s, meta - p))
			return 0;

		s += meta - p;
		p = meta + 1;

		if (isdigit(*p))
			len = simple_strtoul(p, (char **)&p, 10);
		else if (*p == '%') {
			if (*s++ != '%')
				return 0;
			p++;
			continue;
		}

		if (argc >= MAX_OPT_ARGS)
			return 0;

		args[argc].from = s;
		switch (*p++) {
		case 's': {
			size_t str_len = strlen(s);

			if (str_len == 0)
				return 0;
			if (len == -1 || len > str_len)
				len = str_len;
			args[argc].to = s + len;
			break;
		}
		case 'd':
			simple_strtol(s, &args[argc].to, 0);
			goto num;
		case 'u':
			simple_strtoul(s, &args[argc].to, 0);
			goto num;
		case 'o':
			simple_strtoul(s, &args[argc].to, 8);
			goto num;
		case 'x':
			simple_strtoul(s, &args[argc].to, 16);
		num:
			if (args[argc].to == args[argc].from)
				return 0;
			break;
		default:
			return 0;
		}
		s = args[argc].to;
		argc++;
	}
}

static int aml_match_token(char *s, const match_table_t table, substring_t args[])
{
	const struct match_token *p;

	for (p = table; !aml_match_one(s, p->pattern, args); p++)
		;

	return p->token;
}

static int aml_match_int(substring_t *s, int *result)
{
	return match_number(s, result, 0);
}

static char *aml_match_strdup(const substring_t *s)
{
	return kmemdup_nul(s->from, s->to - s->from, GFP_KERNEL);
}

static int parse_options(const char *buffer, struct memtest_option *option)
{
	char *p, *s, *err, *tmp_buffer, *orig_buffer;
	substring_t args[MAX_OPT_ARGS];
	int len;

	if (!buffer)
		return 1;

	tmp_buffer = kstrdup(buffer, GFP_KERNEL);
	if (!tmp_buffer)
		return 1;

	orig_buffer = tmp_buffer;

	len = strlen(tmp_buffer);
	if (len && tmp_buffer[len - 1] == '\n')
		tmp_buffer[len - 1] = '\0';

	while ((p = strsep(&tmp_buffer, " ")) != NULL) {
		int token, r, value, i;

		if (!*p)
			continue;

		token = aml_match_token(p, memtest_tokens, args);
		switch (token) {
		case OPT_MODE:
			s = aml_match_strdup(&args[0]);
			if (!s) {
				err = "bad rw";
				goto error;
			}
			for (i = 0; i < MODE_MAX; i++) {
				if (!strcmp(s, mode_str[i])) {
					option->mode = i;
					break;
				}
			}
			kfree(s);
			if (i == MODE_MAX) {
				err = "bad rw";
				goto error;
			}
			break;
		case OPT_MEM_TYPE:
			s = aml_match_strdup(&args[0]);
			if (!s) {
				err = "bad cache";
				goto error;
			}
			for (i = 0; i < MEM_TYPE_MAX; i++) {
				if (!strcmp(s, mem_type_str[i])) {
					option->mem_type = i;
					break;
				}
			}
			kfree(s);
			if (i == MEM_TYPE_MAX) {
				err = "bad mem_type";
				goto error;
			}
			break;
		case OPT_SIZE_KB:
			r = aml_match_int(&args[0], &value);
			if (r < 0) {
				err = "bad size_kb";
				goto error;
			}
			option->size_kb = value;
			break;
		case OPT_CPUMASK:
			r = aml_match_int(&args[0], &value);
			option->cpumask = value;
			break;
		case OPT_MIN_PFN:
			r = aml_match_hex(&args[0], &value);
			if (r < 0) {
				err = "bad min_pfn";
				goto error;
			}
			option->min_pfn = value;
			break;
		case OPT_MAX_PFN:
			r = aml_match_hex(&args[0], &value);
			if (r < 0) {
				err = "bad max_pfn";
				goto error;
			}
			option->max_pfn = value;
			break;
		case OPT_LOOPS:
			r = aml_match_int(&args[0], &value);
			if (r < 0) {
				err = "bad loops";
				goto error;
			}
			option->loops = value;
			break;
		case OPT_SUBLOOPS:
			r = aml_match_int(&args[0], &value);
			if (r < 0) {
				err = "bad subloops";
				goto error;
			}
			option->subloops = value;
			break;
		case OPT_LOOP_SLEEPMS:
			r = aml_match_int(&args[0], &value);
			if (r < 0) {
				err = "bad loop_sleepms";
				goto error;
			}
			option->loop_sleepms = value;
			break;
		case OPT_DISABLE_IRQ:
			r = aml_match_int(&args[0], &value);
			if (r < 0) {
				err = "bad disable_irq";
				goto error;
			}
			option->disable_irq = value;
			break;
		case OPT_RAND_CNT:
			r = aml_match_int(&args[0], &value);
			if (r < 0) {
				err = "bad rand_cnt";
				goto error;
			}
			option->rand_cnt = value;
			break;
		case OPT_PRINT:
			r = aml_match_int(&args[0], &value);
			if (r < 0) {
				err = "bad print";
				goto error;
			}
			option->print = value;
			break;
		case OPT_STOP:
			memtest_stop = 1;
			break;
		default:
			err = "unknown arg";
			goto error;
		}
	}

	kfree(orig_buffer);
	return 0;
error:
	kfree(orig_buffer);
	show_help(err);
	return 1;
}

static int memtest_trigger_set(const char *buffer, const struct kernel_param *kp)
{
	struct memtest_option *option;

	option = kmalloc(sizeof(*option), GFP_KERNEL);
	if (!option)
		return -1;

	//default
	option->mode = MODE_COPY;
	option->mem_type = MEM_TYPE_NORMAL;
	option->size_kb = 64 * 1024;
	option->cpumask = 0xff;
	option->min_pfn = 0x0;
	option->max_pfn = 0xffffffff;
	option->loops = -1; //deadloop
	option->subloops = SUBLOOPS;
	option->loop_sleepms = 0;
	option->disable_irq = 1;
	option->rand_cnt = 1000;
	option->print = 1;

	if (parse_options(buffer, option))
		return -1;

	if (memtest_stop)
		do_memtest_stop();
	else
		do_memtest(option);

	return 0;
}

static int memtest_trigger_get(char *buffer, const struct kernel_param *kp)
{
	show_help(NULL);
	return 0;
}

static const struct kernel_param_ops memtest_trigger_ops = {
	.set    = memtest_trigger_set,
	.get	= memtest_trigger_get,
};
module_param_cb(memtest_trigger, &memtest_trigger_ops, NULL, 0644);

#define FIVE(m) {m; m; m; m; m; }
#define TEN(m) {FIVE(m) FIVE(m)}
#define FIFTY(m) {TEN(m) TEN(m) TEN(m) TEN(m) TEN(m)}
#define HUNDRED(m) {FIFTY(m) FIFTY(m)}

#define LINE_SIZE 64
#define LINES_PER_PAGE (PAGE_SIZE / LINE_SIZE)
static int seeds[LINES_PER_PAGE] = {0, 40, 52, 2, 44, 26, 15, 59, 63, 20, 62, 31, 9, 25, 41, 18, 45, 1, 19, 6, 39, 46, 12, 17, 10, 38, 61, 16, 56, 7, 55, 35, 13, 21, 34, 23, 4, 36, 29, 50, 33, 8, 53, 60, 47, 22, 54, 49, 32, 28, 57, 43, 14, 24, 37, 11, 48, 5, 42, 51, 3, 27, 58, 30};

static void memtest_random_init(unsigned long addr, int size)
{
	int i, j, lines_num, pages_num, page;
	int *page_id;
	unsigned long c_base, n_base;
	unsigned long curr_addr, next_addr;
//	static int cnt;

	lines_num = size / LINE_SIZE;
	pages_num = lines_num / LINES_PER_PAGE;

	//generate a random page visiting page_id at runtime (Fisher-Yates
	//shuffle with the kernel PRNG) so that page i+1 in the chain is
	//a random page spread over the whole buffer, not i+1 sequentially
	page_id = kmalloc_array(pages_num, sizeof(int), GFP_KERNEL);
	if (page_id) {
		for (i = 0; i < pages_num; i++)
			page_id[i] = i;

		for (i = pages_num - 1; i > 0; i--) {
			u32 r = get_random_u32() % (i + 1);

			page = page_id[i];
			page_id[i] = page_id[r];
			page_id[r] = page;
		}
	}

	//base lmbench thrash_initialize()
	//page_x[line_a] -> page_y[line_b] ....
	for (i = 0; i < pages_num - 1; i++) {
		c_base = addr + page_id[i] * PAGE_SIZE;
		n_base = addr + page_id[i+1] * PAGE_SIZE;

		for (j = 0; j < LINES_PER_PAGE; j++) {
			curr_addr = c_base + seeds[(i + j) % LINES_PER_PAGE] * LINE_SIZE;
			next_addr = n_base + seeds[(i + j + 1) % LINES_PER_PAGE] * LINE_SIZE;
			*(volatile unsigned long *)curr_addr  = next_addr;

//			pr_emerg("[%05d] %lx ->  %lx   (%d ->  %d)\n", cnt++, curr_addr, next_addr, page_id[i], page_id[i+1]);
		}
	}

	c_base = addr + page_id[i] * PAGE_SIZE; //last page
	n_base = addr + page_id[0] * PAGE_SIZE; //first page

	for (j = 0; j < LINES_PER_PAGE; j++) {
		curr_addr = c_base + seeds[(i + j) % LINES_PER_PAGE] * LINE_SIZE;
		next_addr = n_base + seeds[(j + 1) % LINES_PER_PAGE] * LINE_SIZE;
		*(volatile unsigned long *)curr_addr  = next_addr;

//		pr_emerg("[%05d] %lx ->  %lx   (%d ->  %d)\n", cnt++, curr_addr, next_addr, page_id[i], page_id[0]);
	}

	kfree(page_id);
}

static void free_page_list(struct list_head *pages)
{
	struct page *p, *n;

	list_for_each_entry_safe(p, n, pages, lru)
		__free_page(p);

	INIT_LIST_HEAD(pages);
}

static void *memtest_alloc(struct page ***pages, int *page_cnt, int size_kb, unsigned long min_pfn, unsigned long max_pfn, pgprot_t prot)
{
	char *buffer;
	int i, retries, success;
	LIST_HEAD(free_list);

	*page_cnt = size_kb / 4;
	*pages = kzalloc(*page_cnt * sizeof(struct page *), GFP_KERNEL);
	if (!*pages) {
		pr_err("alloc pages failed\n");
		return NULL;
	}

	for (retries = 0, success = 0; success < *page_cnt; retries++) {
		unsigned long pfn;
		struct page *page = alloc_page(GFP_HIGHUSER_MOVABLE);

		if (!page) {
			pr_err("alloc_page() failed, restries=%d success=%d\n", retries, success);
			goto fail;
		}

		pfn = page_to_pfn(page);
		if (pfn >= min_pfn && pfn < max_pfn) {
			(*pages)[success++] = page;
			continue;
		}

		list_add_tail(&page->lru, &free_list);
	}
	free_page_list(&free_list);

	buffer = vmap(*pages, *page_cnt, VM_MAP, prot);

	pr_err("%s/%d: alloc buffer=%px page_cnt=%d retrires=%d success=%d\n",
		current->comm, current->pid,
		buffer,
		*page_cnt,
		retries,
		success);

	return buffer;

fail:
	for (i = 0; i < success; i++)
		__free_page((*pages)[i]);

	free_page_list(&free_list);

	kfree(*pages);
	*pages = NULL;

	return NULL;
}

noinline void memtest_copy(char *buffer, int size_kb)
{
	void *p1, *p2;
	int j;

	p1 = buffer;
	p2 = buffer +  size_kb * 1024 / 2;

	for (j = 0; j < size_kb * 1024 / 2; j += 64) {
		*(volatile unsigned long long *)(p1 + j) = *(volatile unsigned long long *)(p2 + j);
		*(volatile unsigned long long *)(p1 + j + 8) = *(volatile unsigned long long *)(p2 + j + 8);
		*(volatile unsigned long long *)(p1 + j + 16) = *(volatile unsigned long long *)(p2 + j + 16);
		*(volatile unsigned long long *)(p1 + j + 24) = *(volatile unsigned long long *)(p2 + j + 24);
		*(volatile unsigned long long *)(p1 + j + 32) = *(volatile unsigned long long *)(p2 + j + 32);
		*(volatile unsigned long long *)(p1 + j + 40) = *(volatile unsigned long long *)(p2 + j + 40);
		*(volatile unsigned long long *)(p1 + j + 48) = *(volatile unsigned long long *)(p2 + j + 48);
		*(volatile unsigned long long *)(p1 + j + 56) = *(volatile unsigned long long *)(p2 + j + 56);
	}
}

noinline void memtest_readonly(char *buffer, int size_kb)
{
	int j;

	for (j = 0; j < size_kb * 1024; j += 64) {
		*(volatile unsigned long long *)(buffer + j);
		*(volatile unsigned long long *)(buffer + j + 8);
		*(volatile unsigned long long *)(buffer + j + 16);
		*(volatile unsigned long long *)(buffer + j + 24);
		*(volatile unsigned long long *)(buffer + j + 32);
		*(volatile unsigned long long *)(buffer + j + 40);
		*(volatile unsigned long long *)(buffer + j + 48);
		*(volatile unsigned long long *)(buffer + j + 56);
	}
}

noinline void memtest_writeonly(char *buffer, int size_kb)
{
	int j;

	for (j = 0; j < size_kb * 1024; j += 64) {
		*(volatile unsigned long long *)(buffer + j) = 1;
		*(volatile unsigned long long *)(buffer + j + 8) = 1;
		*(volatile unsigned long long *)(buffer + j + 16) = 1;
		*(volatile unsigned long long *)(buffer + j + 24) = 1;
		*(volatile unsigned long long *)(buffer + j + 32) = 1;
		*(volatile unsigned long long *)(buffer + j + 40) = 1;
		*(volatile unsigned long long *)(buffer + j + 48) = 1;
		*(volatile unsigned long long *)(buffer + j + 56) = 1;
	}
}

static int memtest_thread(void *data)
{
	char *buffer;
	int i, j, loop;
	unsigned long long *delta;
	int mode, size_kb, loops, loop_subloops, loop_sleepms, rand_cnt, disable_irq;
	int min_pfn, max_pfn;
	struct page **pages;
	int page_cnt;
	volatile unsigned long *p = NULL;
	pgprot_t prot = PAGE_KERNEL;
	int print;
	int mem_type;
	int cpumask;
	struct memtest_option *option = data;
	struct memtest_task *mtask;

	mtask = kmalloc(sizeof(*mtask), GFP_KERNEL);
	if (!mtask) {
		kfree(option);
		atomic_dec(&memtest_pending);
		return 0;
	}
	mtask->task = current;

	//register in the active task list so that memtest_stop can
	//wait for this thread to exit
	mutex_lock(&memtest_lock);
	list_add_tail(&mtask->list, &memtest_tasks);
	mutex_unlock(&memtest_lock);

	mode = option->mode;
	size_kb = option->size_kb;
	cpumask = option->cpumask;
	min_pfn = option->min_pfn;
	max_pfn = option->max_pfn;
	loops = option->loops;
	loop_subloops = option->subloops;
	loop_sleepms = option->loop_sleepms;
	rand_cnt = option->rand_cnt;
	disable_irq = option->disable_irq;
	print = option->print;
	mem_type = option->mem_type;
	//align to 4K
	size_kb = ALIGN(size_kb, 4);

	kfree(option);

	if (loop_subloops < 1)
		loop_subloops = 1;
	else if (loop_subloops > MAX_SUBLOOPS)
		loop_subloops = MAX_SUBLOOPS;

	delta = kmalloc_array(MAX_SUBLOOPS, sizeof(*delta), GFP_KERNEL); //max 800KB
	if (!delta)
		goto out;

	//a stop was written before this thread got to run: exit
	//immediately so the stop setter is not left waiting
	if (memtest_stop)
		goto out;

	pr_emerg("%s/%d: rw=%s cache=%s cpumask=0x%x size_kb=%d min_pfn=0x%x max_pfn=0x%x loops=%d loop_subloops=%d loop_sleepms=%d rand_cnt=%d disable_irq=%d print=%d\n",
		current->comm, current->pid,
		mode_str[mode],
		mem_type_str[mem_type],
		cpumask,
		size_kb,
		min_pfn, max_pfn,
		loops,
		loop_subloops,
		loop_sleepms,
		rand_cnt,
		disable_irq,
		print);

	switch (mem_type) {
	case MEM_TYPE_NORMAL:
		prot = PAGE_KERNEL;
		break;
	case MEM_TYPE_NONCACHED:
		prot = pgprot_noncached(PAGE_KERNEL);
		break;
	case MEM_TYPE_WRITECOMBINE:
		prot = pgprot_writecombine(PAGE_KERNEL);
		break;
	default:
		break;
	}

	buffer = memtest_alloc(&pages, &page_cnt, size_kb, min_pfn, max_pfn, prot);
	if (!buffer) {
		pr_err("memtest_alloc failed\n");
		goto out;
	}

	if (mode == MODE_RANDOM) {
		//flush cache all first, or noncached memory access may will not take effect
#ifdef CONFIG_ARM64
		struct platform_device *pdev;

		pdev = platform_device_register_data(NULL, "memtest", -1, NULL, 0);
		dma_sync_single_for_device(&pdev->dev, (dma_addr_t)buffer, size_kb * 1024, DMA_FROM_DEVICE);
		platform_device_unregister(pdev);
#endif
#ifdef CONFIG_ARM
		struct platform_device *pdev;
		struct page *page;
		int i;

		pdev = platform_device_register_data(NULL, "memtest", -1, NULL, 0);
		for (i = 0; i < size_kb * 1024 / PAGE_SIZE; i++) {
			page = vmalloc_to_page(buffer + i * PAGE_SIZE);
			if (page)
				dma_map_page(&pdev->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
		}
		platform_device_unregister(pdev);
#endif

		memtest_random_init((unsigned long)buffer, size_kb * 1024);
		p = (volatile unsigned long *)buffer;
	}

	loop = 0;
do_loop:

	for (i = 0; i < loop_subloops; i++) {
		unsigned long flags;
		unsigned long long start;

		if (disable_irq)
			local_irq_save(flags);

		start = sched_clock();
		if (mode == MODE_COPY) {
			memtest_copy(buffer, size_kb);
		} else if (mode == MODE_RDONLY) {
			memtest_readonly(buffer, size_kb);
		} else if (mode == MODE_WRONLY) {
			memtest_writeonly(buffer, size_kb);
		} else if (mode == MODE_RANDOM) {
			for (j = 0; j < rand_cnt; j++) {
//				HUNDRED(pr_emerg("%px = %lx\n", p, *p); p = (volatile unsigned long *)(*p););
				HUNDRED(p = (volatile unsigned long *)(*p););
			}
		}
		delta[i] = sched_clock() - start;

		if (disable_irq)
			local_irq_restore(flags);
	}

	if (print) {
		unsigned long long min, max, avg, min_tmp, max_tmp, avg_tmp;

		min = -1ULL;
		max = 0;
		avg = 0;

		for (i = 0; i < loop_subloops; i++) {
			if (delta[i] < min)
				min = delta[i];

			if (delta[i] > max)
				max = delta[i];

			avg += delta[i];
		}

		avg = div_u64(avg, loop_subloops);

		min_tmp = min;
		max_tmp = max;
		avg_tmp = avg;

		if (mode == MODE_RANDOM)
			pr_err("stat: pid=%d rw=%s cache=%s cpumask=0x%x size_kb=%d min=%llu.%06ums max=%llu.%06ums avg=%llu.%06ums latency:%lluns\n",
				current->pid,
				mode_str[mode], mem_type_str[mem_type],
				cpumask,
				size_kb,
				div_u64(min, 1000000), do_div(min_tmp, 1000000),
				div_u64(max, 1000000), do_div(max_tmp, 1000000),
				div_u64(avg, 1000000), do_div(avg_tmp, 1000000),
				div_u64(min, rand_cnt * 100));
		else
			pr_err("stat: pid=%d rw=%s cache=%s cpumask=0x%x size_kb=%d min=%llu.%06ums max=%llu.%06ums avg=%llu.%06ums bandwidth:%lluMB\n",
				current->pid,
				mode_str[mode], mem_type_str[mem_type],
				cpumask,
				size_kb,
				div_u64(min, 1000000), do_div(min_tmp, 1000000),
				div_u64(max, 1000000), do_div(max_tmp, 1000000),
				div_u64(avg, 1000000), do_div(avg_tmp, 1000000),
				div_u64(size_kb * 1000000ULL, min));
	}

	if (memtest_stop)
		goto free;

	if (loop_sleepms)
		//woken early when memtest_stop is set, so the stop
		//setter does not have to wait out the sleep
		wait_event_interruptible_timeout(memtest_waitq, memtest_stop,
						 loop_sleepms * HZ / 1000);

	loop++;
	if (loops == -1 || loop < loops)
		goto do_loop;

free:
	pr_emerg("%s/%d: stop!\n", current->comm, current->pid);
	vfree(buffer);
	for (i = 0; i < page_cnt; i++)
		__free_page(pages[i]);

	kfree(pages);
	kfree(delta);
out:
	//remove self from the active task list and wake up anyone
	//waiting in memtest_stop for all threads to exit
	mutex_lock(&memtest_lock);
	list_del(&mtask->list);
	mutex_unlock(&memtest_lock);
	wake_up(&memtest_waitq);

	kfree(mtask);
	atomic_dec(&memtest_pending);

	return 0;
}

static void do_memtest(struct memtest_option *option)
{
	struct task_struct *p;
	char buf[64];

	//the stop check and the pending count must be done atomically
	//under memtest_lock: kthread_create() can sleep, and a stop
	//triggered while it sleeps must either be rejected here or see
	//the pending count and wait for this thread
	mutex_lock(&memtest_lock);
	if (memtest_stop) {
		mutex_unlock(&memtest_lock);
		pr_err("memtest: stop in progress, trigger ignored\n");
		kfree(option);
		return;
	}
	//count the thread before it can run, so that a concurrent stop
	//waits for it even if it has not registered itself yet
	atomic_inc(&memtest_pending);
	mutex_unlock(&memtest_lock);

	sprintf(buf, "mem_%s", mode_str[option->mode]);

	p = kthread_create(memtest_thread, option, buf);
	if (!p) {
		atomic_dec(&memtest_pending);
		kfree(option);
		return;
	}

	kthread_bind_mask(p, (const struct cpumask *)&option->cpumask);

	wake_up_process(p);
}