Merge "sched: Ensure proper task migration when a CPU is isolated"

2 files changed, 44 insertions, 10 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 13990fa6f9cf..ee708909dc17 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5464,6 +5464,37 @@ static struct task_struct fake_task = {
 };
 
 /*
+ * Remove a task from the runqueue and pretend that it's migrating. This
+ * should prevent migrations for the detached task and disallow further
+ * changes to tsk_cpus_allowed.
+ */
+static void
+detach_one_task(struct task_struct *p, struct rq *rq, struct list_head *tasks)
+{
+	lockdep_assert_held(&rq->lock);
+
+	p->on_rq = TASK_ON_RQ_MIGRATING;
+	deactivate_task(rq, p, 0);
+	list_add(&p->se.group_node, tasks);
+}
+
+static void attach_tasks(struct list_head *tasks, struct rq *rq)
+{
+	struct task_struct *p;
+
+	lockdep_assert_held(&rq->lock);
+
+	while (!list_empty(tasks)) {
+		p = list_first_entry(tasks, struct task_struct, se.group_node);
+		list_del_init(&p->se.group_node);
+
+		BUG_ON(task_rq(p) != rq);
+		activate_task(rq, p, 0);
+		p->on_rq = TASK_ON_RQ_QUEUED;
+	}
+}
+
+/*
  * Migrate all tasks (not pinned if pinned argument say so) from the rq,
  * sleeping tasks will be migrated by try_to_wake_up()->select_task_rq().
  *
@@ -5477,6 +5508,7 @@ static void migrate_tasks(struct rq *dead_rq, bool migrate_pinned_tasks)
 	struct task_struct *next, *stop = rq->stop;
 	int dest_cpu;
 	unsigned int num_pinned_kthreads = 1; /* this thread */
+	LIST_HEAD(tasks);
 	cpumask_t avail_cpus;
 
 	cpumask_andnot(&avail_cpus, cpu_online_mask, cpu_isolated_mask);
@@ -5501,12 +5533,10 @@ static void migrate_tasks(struct rq *dead_rq, bool migrate_pinned_tasks)
 
 	for (;;) {
 		/*
-		 * There's this thread running + pinned threads, bail when
-		 * that's the only remaining threads.
+		 * There's this thread running, bail when that's the only
+		 * remaining thread.
 		 */
-		if ((migrate_pinned_tasks && rq->nr_running == 1) ||
-		   (!migrate_pinned_tasks &&
-		    rq->nr_running <= num_pinned_kthreads))
+		if (rq->nr_running == 1)
 			break;
 
 		/*
@@ -5519,8 +5549,9 @@ static void migrate_tasks(struct rq *dead_rq, bool migrate_pinned_tasks)
 
 		if (!migrate_pinned_tasks && next->flags & PF_KTHREAD &&
 			!cpumask_intersects(&avail_cpus, &next->cpus_allowed)) {
-			lockdep_unpin_lock(&rq->lock);
+			detach_one_task(next, rq, &tasks);
 			num_pinned_kthreads += 1;
+			lockdep_unpin_lock(&rq->lock);
 			continue;
 		}
 
@@ -5568,6 +5599,9 @@ static void migrate_tasks(struct rq *dead_rq, bool migrate_pinned_tasks)
 	}
 
 	rq->stop = stop;
+
+	if (num_pinned_kthreads > 1)
+		attach_tasks(&tasks, rq);
 }
 
 static void set_rq_online(struct rq *rq);
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 12a04f30ef77..52edd6b158ed 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1970,11 +1970,11 @@ retry:
 		goto retry;
 	}
 
-	deactivate_task(rq, next_task, 0);
 	next_task->on_rq = TASK_ON_RQ_MIGRATING;
+	deactivate_task(rq, next_task, 0);
 	set_task_cpu(next_task, lowest_rq->cpu);
-	next_task->on_rq = TASK_ON_RQ_QUEUED;
 	activate_task(lowest_rq, next_task, 0);
+	next_task->on_rq = TASK_ON_RQ_QUEUED;
 	ret = 1;
 
 	resched_curr(lowest_rq);
@@ -2226,11 +2226,11 @@ static void pull_rt_task(struct rq *this_rq)
 
 			resched = true;
 
-			deactivate_task(src_rq, p, 0);
 			p->on_rq = TASK_ON_RQ_MIGRATING;
+			deactivate_task(src_rq, p, 0);
 			set_task_cpu(p, this_cpu);
-			p->on_rq = TASK_ON_RQ_QUEUED;
 			activate_task(this_rq, p, 0);
+			p->on_rq = TASK_ON_RQ_QUEUED;
 			/*
 			 * We continue with the search, just in
 			 * case there's an even higher prio task