cpufreq: cpu-boost: Add cpu-boost driver

When certain bursty and important events take place, it might take a while
for the current cpufreq governor to notice the new load and react to it.
That would result in poor user experience. To alleviate this, the cpu-boost
driver boosts the frequency of a CPU for a short duration to maintain good
user experience while the governor catches up.

Specifically, this commit deals with ensuring that when "important" tasks
migrate from a fast CPU to a slow CPU, the frequency of the slow CPU is
boosted to be at least as high as the fast CPU for a short duration.

Since this driver enforces the boost by hooking into standard cpufreq
ADJUST notifiers, it has several advantages:
- More portable across kernel versions where the cpufreq internals might
  have been rewritten.
- Governor agnostic and hence works with multiple governors like
  conservative, ondemand, interactive, etc.
- Does not affect the sampling period/logic of existing governors.
- Can have the boost period adjusted independent of governor sampling
  period.

Change-Id: Ibd814a20043d0aba64ee7637a4a79b9ffa1b0991
Signed-off-by: Saravana Kannan <skannan@codeaurora.org>
[rameezmustafa@codeaurora.org]: Port to msm-3.18]
Signed-off-by: Syed Rameez Mustafa <rameezmustafa@codeaurora.org>
[joonwoop@codeaurora.org: fixed conflict in drivers/cpufreq/Kconfig.]
Signed-off-by: Joonwoo Park <joonwoop@codeaurora.org>

3 files changed, 194 insertions, 0 deletions

diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig
index 75f63efd7b43..e3ddea30ac61 100644
--- a/drivers/cpufreq/Kconfig
+++ b/drivers/cpufreq/Kconfig
@@ -222,6 +222,17 @@ config CPUFREQ_DT
 
 	  If in doubt, say N.
 
+config CPU_BOOST
+	tristate "Event base short term CPU freq boost"
+	depends on CPU_FREQ
+	help
+	  This driver boosts the frequency of one or more CPUs based on
+	  various events that might occur in the system. As of now, the
+	  events it reacts to are:
+	  - Migration of important threads from one CPU to another.
+
+	  If in doubt, say N.
+
 if X86
 source "drivers/cpufreq/Kconfig.x86"
 endif
diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
index b02ae1463a15..d6e626f3447c 100644
--- a/drivers/cpufreq/Makefile
+++ b/drivers/cpufreq/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_CPU_FREQ_GOV_ONDEMAND)	+= cpufreq_ondemand.o
 obj-$(CONFIG_CPU_FREQ_GOV_CONSERVATIVE)	+= cpufreq_conservative.o
 obj-$(CONFIG_CPU_FREQ_GOV_INTERACTIVE)	+= cpufreq_interactive.o
 obj-$(CONFIG_CPU_FREQ_GOV_COMMON)		+= cpufreq_governor.o
+obj-$(CONFIG_CPU_BOOST)			+= cpu-boost.o
 
 obj-$(CONFIG_CPUFREQ_DT)		+= cpufreq-dt.o
 
diff --git a/drivers/cpufreq/cpu-boost.c b/drivers/cpufreq/cpu-boost.c
new file mode 100644
index 000000000000..91077f531b9e
--- /dev/null
+++ b/drivers/cpufreq/cpu-boost.c
@@ -0,0 +1,182 @@
+/*
+ * Copyright (c) 2013, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt) "cpu-boost: " fmt
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/notifier.h>
+#include <linux/cpufreq.h>
+#include <linux/sched.h>
+#include <linux/jiffies.h>
+#include <linux/kthread.h>
+#include <linux/moduleparam.h>
+
+struct cpu_sync {
+	struct task_struct *thread;
+	wait_queue_head_t sync_wq;
+	struct delayed_work boost_rem;
+	int cpu;
+	spinlock_t lock;
+	bool pending;
+	int src_cpu;
+	unsigned int boost_min;
+};
+
+static DEFINE_PER_CPU(struct cpu_sync, sync_info);
+static struct workqueue_struct *boost_rem_wq;
+
+static unsigned int boost_ms;
+module_param(boost_ms, uint, 0644);
+
+/*
+ * The CPUFREQ_ADJUST notifier is used to override the current policy min to
+ * make sure policy min >= boost_min. The cpufreq framework then does the job
+ * of enforcing the new policy.
+ */
+static int boost_adjust_notify(struct notifier_block *nb, unsigned long val,
+				void *data)
+{
+	struct cpufreq_policy *policy = data;
+	unsigned int cpu = policy->cpu;
+	struct cpu_sync *s = &per_cpu(sync_info, cpu);
+	unsigned int min = s->boost_min;
+
+	if (val != CPUFREQ_ADJUST)
+		return NOTIFY_OK;
+
+	if (min == 0)
+		return NOTIFY_OK;
+
+	pr_debug("CPU%u policy min before boost: %u kHz\n",
+		 cpu, policy->min);
+	pr_debug("CPU%u boost min: %u kHz\n", cpu, min);
+
+	cpufreq_verify_within_limits(policy, min, UINT_MAX);
+
+	pr_debug("CPU%u policy min after boost: %u kHz\n",
+		 cpu, policy->min);
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block boost_adjust_nb = {
+	.notifier_call = boost_adjust_notify,
+};
+
+static void do_boost_rem(struct work_struct *work)
+{
+	struct cpu_sync *s = container_of(work, struct cpu_sync,
+						boost_rem.work);
+
+	pr_debug("Removing boost for CPU%d\n", s->cpu);
+	s->boost_min = 0;
+	/* Force policy re-evaluation to trigger adjust notifier. */
+	cpufreq_update_policy(s->cpu);
+}
+
+static int boost_mig_sync_thread(void *data)
+{
+	int dest_cpu = (int) data;
+	int src_cpu, ret;
+	struct cpu_sync *s = &per_cpu(sync_info, dest_cpu);
+	struct cpufreq_policy dest_policy;
+	struct cpufreq_policy src_policy;
+	unsigned long flags;
+
+	while (1) {
+		wait_event(s->sync_wq, s->pending || kthread_should_stop());
+
+		if (kthread_should_stop())
+			break;
+
+		spin_lock_irqsave(&s->lock, flags);
+		s->pending = false;
+		src_cpu = s->src_cpu;
+		spin_unlock_irqrestore(&s->lock, flags);
+
+		ret = cpufreq_get_policy(&src_policy, src_cpu);
+		if (ret)
+			continue;
+
+		ret = cpufreq_get_policy(&dest_policy, dest_cpu);
+		if (ret)
+			continue;
+
+		if (dest_policy.cur >= src_policy.cur) {
+			pr_debug("No sync. CPU%d@%dKHz >= CPU%d@%dKHz\n",
+				 dest_cpu, dest_policy.cur,
+				 src_cpu, src_policy.cur);
+			continue;
+		}
+
+		cancel_delayed_work_sync(&s->boost_rem);
+		s->boost_min = src_policy.cur;
+		/* Force policy re-evaluation to trigger adjust notifier. */
+		cpufreq_update_policy(dest_cpu);
+		queue_delayed_work_on(s->cpu, boost_rem_wq,
+			&s->boost_rem, msecs_to_jiffies(boost_ms));
+	}
+
+	return 0;
+}
+
+static int boost_migration_notify(struct notifier_block *nb,
+				unsigned long dest_cpu, void *arg)
+{
+	unsigned long flags;
+	struct cpu_sync *s = &per_cpu(sync_info, dest_cpu);
+
+	if (!boost_ms)
+		return NOTIFY_OK;
+
+	pr_debug("Migration: CPU%d --> CPU%d\n", (int) arg, (int) dest_cpu);
+	spin_lock_irqsave(&s->lock, flags);
+	s->pending = true;
+	s->src_cpu = (int) arg;
+	spin_unlock_irqrestore(&s->lock, flags);
+	wake_up(&s->sync_wq);
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block boost_migration_nb = {
+	.notifier_call = boost_migration_notify,
+};
+
+static int cpu_boost_init(void)
+{
+	int cpu;
+	struct cpu_sync *s;
+
+	cpufreq_register_notifier(&boost_adjust_nb, CPUFREQ_POLICY_NOTIFIER);
+
+	boost_rem_wq = alloc_workqueue("cpuboost_rem_wq", WQ_HIGHPRI, 0);
+	if (!boost_rem_wq)
+		return -EFAULT;
+
+	for_each_possible_cpu(cpu) {
+		s = &per_cpu(sync_info, cpu);
+		s->cpu = cpu;
+		init_waitqueue_head(&s->sync_wq);
+		spin_lock_init(&s->lock);
+		INIT_DELAYED_WORK(&s->boost_rem, do_boost_rem);
+		s->thread = kthread_run(boost_mig_sync_thread, (void *)cpu,
+					"boost_sync/%d", cpu);
+	}
+	atomic_notifier_chain_register(&migration_notifier_head,
+					&boost_migration_nb);
+
+	return 0;
+}
+late_initcall(cpu_boost_init);