kernel26-alx/patches-2.6.21-r15/0006-2.6.21-sched-idleprio-2.3.patch

Add the SCHED_IDLEPRIO scheduling policy. Tasks set to this policy are only
given cpu time if no other tasks at all wish to have cpu time thus running
effectively at idle priority. If semaphores or mutexes are held, or the
system is going into suspend, schedule them as SCHED_NORMAL nice 19.

Signed-off-by: Con Kolivas <kernel@kolivas.org>
---
 include/linux/sched.h |    5 +
 kernel/sched.c        |  145 ++++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 138 insertions(+), 12 deletions(-)

Index: linux-2.6.21-ck2/include/linux/sched.h
===================================================================
--- linux-2.6.21-ck2.orig/include/linux/sched.h	2007-05-14 19:30:31.000000000 +1000
+++ linux-2.6.21-ck2/include/linux/sched.h	2007-05-14 19:30:31.000000000 +1000
@@ -35,10 +35,11 @@
 #define SCHED_RR		2
 #define SCHED_BATCH		3
 #define SCHED_ISO		4
+#define SCHED_IDLEPRIO		5
 
 #ifdef __KERNEL__
 
-#define SCHED_MAX		SCHED_ISO
+#define SCHED_MAX		SCHED_IDLEPRIO
 #define SCHED_RANGE(policy)	((policy) <= SCHED_MAX)
 
 struct sched_param {
@@ -537,6 +538,7 @@ struct signal_struct {
 					(policy) == SCHED_RR)
 #define has_rt_policy(p)	unlikely(is_rt_policy((p)->policy))
 #define iso_task(p)		unlikely((p)->policy == SCHED_ISO)
+#define idleprio_task(p)	unlikely((p)->policy == SCHED_IDLEPRIO)
 
 /*
  * Some day this will be a full-fledged user tracking system..
@@ -1173,6 +1175,7 @@ static inline void put_task_struct(struc
 #define PF_ISOREF	0x04000000	/* SCHED_ISO task has used up quota */
 #define PF_MEMPOLICY	0x10000000	/* Non-default NUMA mempolicy */
 #define PF_MUTEX_TESTER	0x20000000	/* Thread belongs to the rt mutex tester */
+#define PF_NONSLEEP	0x40000000	/* Waiting on in-kernel activity */
 
 /*
  * Only the _current_ task can read/write to tsk->flags, but other
Index: linux-2.6.21-ck2/kernel/sched.c
===================================================================
--- linux-2.6.21-ck2.orig/kernel/sched.c	2007-05-14 19:30:31.000000000 +1000
+++ linux-2.6.21-ck2/kernel/sched.c	2007-05-14 19:30:31.000000000 +1000
@@ -144,7 +144,7 @@ struct rq;
  */
 struct prio_array {
 	/* Tasks queued at each priority */
-	struct list_head queue[MAX_PRIO];
+	struct list_head queue[MAX_PRIO + 1];
 
 	/*
 	 * The bitmap of priorities queued for this array. While the expired
@@ -201,7 +201,7 @@ struct rq {
 	unsigned long next_balance;
 	struct mm_struct *prev_mm;
 
-	struct prio_array *active, *expired, arrays[2];
+	struct prio_array *active, *expired, *idleprio, arrays[2];
 	unsigned long *dyn_bitmap, *exp_bitmap;
 
 	/*
@@ -215,6 +215,8 @@ struct rq {
 	unsigned long iso_ticks;
 	unsigned short iso_refractory;
 
+	/* Number of idleprio tasks running */
+	unsigned long nr_idleprio;
 	atomic_t nr_iowait;
 
 #ifdef CONFIG_SMP
@@ -652,6 +654,17 @@ sched_info_switch(struct task_struct *pr
 #define sched_info_switch(t, next)	do { } while (0)
 #endif /* CONFIG_SCHEDSTATS || CONFIG_TASK_DELAY_ACCT */
 
+static int idleprio_suitable(struct task_struct *p)
+{
+	return (!p->mutexes_held && !freezing(p) && !signal_pending(p) &&
+		!(p->flags & (PF_NONSLEEP | PF_EXITING)));
+}
+
+static int idleprio(const struct task_struct *p)
+{
+	return (p->prio == MAX_PRIO);
+}
+
 static inline int task_queued(struct task_struct *task)
 {
 	return !list_empty(&task->run_list);
@@ -668,7 +681,9 @@ static inline void set_dynamic_bit(struc
 static void dequeue_task(struct task_struct *p, struct rq *rq)
 {
 	list_del_init(&p->run_list);
-	if (list_empty(p->array->queue + p->prio))
+	if (idleprio_task(p) && idleprio(p))
+		rq->nr_idleprio--;
+	else if (list_empty(p->array->queue + p->prio))
 		__clear_bit(p->prio, p->array->prio_bitmap);
 }
 
@@ -809,6 +824,8 @@ static inline int isoprio_suitable(struc
 	return !(p->flags & PF_ISOREF);
 }
 
+static int task_timeslice(struct task_struct *p);
+
 /*
  * recalc_task_prio determines what priority a non rt_task will be
  * queued at. If the task has already been running during this runqueue's
@@ -842,6 +859,30 @@ static void recalc_task_prio(struct task
 			/* Just about to be demoted to SCHED_NORMAL */
 			p->time_slice = 0;
 		}
+	} else if (idleprio_task(p)) {
+		if (idleprio_suitable(p)) {
+			/*
+			 * If suitable idleprio_tasks are queued at MAX_PRIO
+			 * only on the idleprio array. Their time_slice is
+			 * their full task_timeslice as they cooperatively
+			 * multitask.
+			 */
+			p->prio = p->normal_prio = MAX_PRIO;
+			p->array = rq->idleprio;
+			if (p->time_slice <= 0)
+				p->time_slice = task_timeslice(p);
+			return;
+		}
+		/*
+		 * If unsuitable idleprio_tasks are queued equivalent to
+		 * nice 19 tasks on the expired array.
+		 */
+		p->flags &= ~PF_NONSLEEP;
+		p->prio = p->normal_prio = MAX_PRIO - 1;
+		p->array = rq->expired;
+		if (p->time_slice <= 0 || p->time_slice > p->quota)
+			p->time_slice = p->quota;
+		return;
 	}
 
 	update_if_moved(p, rq);
@@ -878,6 +919,8 @@ static inline void __enqueue_task(struct
 	else
 		recalc_task_prio(p, rq);
 
+	if (idleprio_task(p) && idleprio(p))
+		rq->nr_idleprio++;
 	sched_info_queued(p);
 	set_dynamic_bit(p, rq);
 }
@@ -942,6 +985,8 @@ static int task_timeslice(struct task_st
 
 static void set_load_weight(struct task_struct *p)
 {
+	int load_weight;
+
 	if (has_rt_policy(p)) {
 #ifdef CONFIG_SMP
 		if (p == task_rq(p)->migration_thread)
@@ -950,12 +995,19 @@ static void set_load_weight(struct task_
 			 * Giving its load any weight will skew balancing
 			 * adversely.
 			 */
-			p->load_weight = 0;
+			load_weight = 0;
 		else
 #endif
-			p->load_weight = RTPRIO_TO_LOAD_WEIGHT(p->rt_priority);
+			load_weight = RTPRIO_TO_LOAD_WEIGHT(p->rt_priority);
 	} else
-		p->load_weight = task_timeslice(p);
+		load_weight = task_timeslice(p);
+	/*
+	 * idleprio tasks have much lower weight than SCHED_NORMAL tasks but
+	 * still need to be weighted to allow balancing to occur.
+	 */
+	if (likely(!idleprio_task(p)))
+		load_weight *= PRIO_RANGE;
+	p->load_weight = load_weight;
 }
 
 static inline void
@@ -1653,6 +1705,14 @@ out_activate:
 out_running:
 	p->state = TASK_RUNNING;
 out:
+	/*
+	 * Special case when freezing we need to reschedule idleprio tasks
+	 * as SCHED_NORMAL or else they'll never freeze
+	 */
+	if (idleprio_task(p) && freezing(p) && idleprio(p)) {
+		dequeue_task(p, rq);
+		enqueue_task(p, rq);
+	}
 	task_rq_unlock(rq, &flags);
 
 	return success;
@@ -2263,7 +2323,9 @@ skip_bitmap:
 		idx = sched_find_first_bit(array->prio_bitmap);
 	else
 		idx = find_next_bit(array->prio_bitmap, MAX_PRIO, idx);
-	if (idx >= MAX_PRIO) {
+	if (idx == MAX_PRIO) {
+		if (array == busiest->idleprio && busiest->nr_idleprio)
+			goto found_idleprio;
 		if (array == busiest->expired) {
 			array = busiest->active;
 			goto new_array;
@@ -2271,6 +2333,7 @@ skip_bitmap:
 		goto out;
 	}
 
+found_idleprio:
 	head = array->queue + idx;
 	curr = head->prev;
 skip_queue:
@@ -2292,6 +2355,17 @@ skip_queue:
 		best_prio_seen |= idx == best_prio;
 		if (curr != head)
 			goto skip_queue;
+		if (idx == MAX_PRIO) {
+			/*
+			 * Occurs either when balancing idleprio tasks or
+			 * there really are no more tasks to find.
+			 */
+			if (array == busiest->expired) {
+				array = busiest->active;
+				goto new_array;
+			}
+			goto out;
+		}
 		idx++;
 		goto skip_bitmap;
 	}
@@ -2309,6 +2383,13 @@ skip_queue:
 			this_best_prio = idx;
 		if (curr != head)
 			goto skip_queue;
+		if (idx == MAX_PRIO) {
+			if (array == busiest->expired) {
+				array = busiest->active;
+				goto new_array;
+			}
+			goto out;
+		}
 		idx++;
 		goto skip_bitmap;
 	}
@@ -3136,7 +3217,7 @@ void account_user_time(struct task_struc
 
 	/* Add user time to cpustat. */
 	tmp = cputime_to_cputime64(cputime);
-	if (TASK_NICE(p) > 0)
+	if (TASK_NICE(p) > 0 || idleprio_task(p))
 		cpustat->nice = cputime64_add(cpustat->nice, tmp);
 	else
 		cpustat->user = cputime64_add(cpustat->user, tmp);
@@ -3351,6 +3432,28 @@ static void reset_prio_levels(struct rq 
 }
 
 /*
+ * Only tasks running are SCHED_IDLEPRIO. Set the active array to the
+ * idleprio array and if it isn't already active
+ */
+static struct task_struct *next_idleprio_task(struct rq *rq)
+{
+	struct prio_array *array = rq->active;
+	struct list_head *queue;
+
+	if (array != rq->idleprio) {
+		rq->active = rq->idleprio;
+		rq->expired = array;
+		array = rq->active;
+		rq->exp_bitmap = rq->expired->prio_bitmap;
+		rq->dyn_bitmap = rq->active->prio_bitmap;
+	}
+	rq->prio_rotation++;
+	reset_prio_levels(rq);
+	queue = array->queue + MAX_PRIO;
+	return list_entry(queue->next, struct task_struct, run_list);
+}
+
+/*
  * next_dynamic_task finds the next suitable dynamic task.
  */
 static inline struct task_struct *next_dynamic_task(struct rq *rq, int idx)
@@ -3361,6 +3464,8 @@ static inline struct task_struct *next_d
 	int nstatic;
 
 retry:
+	if (unlikely(rq->nr_running == rq->nr_idleprio))
+		return next_idleprio_task(rq);
 	if (idx >= MAX_PRIO) {
 		/* There are no more tasks in the active array. Swap arrays */
 		array = rq->expired;
@@ -3451,8 +3556,10 @@ need_resched_nonpreemptible:
 				unlikely(signal_pending(prev))))
 			prev->state = TASK_RUNNING;
 		else {
-			if (prev->state == TASK_UNINTERRUPTIBLE)
+			if (prev->state == TASK_UNINTERRUPTIBLE) {
+				prev->flags |= PF_NONSLEEP;
 				rq->nr_uninterruptible++;
+			}
 			deactivate_task(prev, rq);
 		}
 	}
@@ -3994,7 +4101,8 @@ void set_user_nice(struct task_struct *p
 		 * If the task increased its priority or is running and
 		 * lowered its priority, then reschedule its CPU:
 		 */
-		if (delta < 0 || (delta > 0 && task_running(rq, p)))
+		if (delta < 0 || ((delta > 0 || idleprio_task(p)) &&
+		    task_running(rq, p)))
 			resched_task(rq->curr);
 	}
 out_unlock:
@@ -4195,6 +4303,11 @@ recheck:
 			return -EPERM;
 	}
 
+	if (!(p->mm) && policy == SCHED_IDLEPRIO) {
+		/* Don't allow kernel threads to be SCHED_IDLEPRIO. */
+		return -EINVAL;
+	}
+
 	retval = security_task_setscheduler(p, policy, param);
 	if (retval)
 		return retval;
@@ -4520,12 +4633,18 @@ asmlinkage long sys_sched_yield(void)
 		struct prio_array *old_array = p->array;
 		int old_prio = p->prio;
 
+		if (idleprio_task(p)) {
+			dequeue_task(p, rq);
+			enqueue_task(p, rq);
+			goto out_release;
+		}
 		/* p->prio will be updated in requeue_task via queue_expired */
 		if (!rt_task(p))
 			p->array = rq->expired;
 		requeue_task(p, rq, old_array, old_prio);
 	}
 
+out_release:
 	/*
 	 * Since we are going to call schedule() anyway, there's
 	 * no need to preempt or enable interrupts:
@@ -4678,6 +4797,7 @@ asmlinkage long sys_sched_get_priority_m
 	case SCHED_NORMAL:
 	case SCHED_BATCH:
 	case SCHED_ISO:
+	case SCHED_IDLEPRIO:
 		ret = 0;
 		break;
 	}
@@ -4703,6 +4823,7 @@ asmlinkage long sys_sched_get_priority_m
 	case SCHED_NORMAL:
 	case SCHED_BATCH:
 	case SCHED_ISO:
+	case SCHED_IDLEPRIO:
 		ret = 0;
 	}
 	return ret;
@@ -6812,8 +6933,10 @@ void __init sched_init(void)
 		lockdep_set_class(&rq->lock, &rq->rq_lock_key);
 		rq->iso_ticks = 0;
 		rq->nr_running = 0;
+		rq->nr_idleprio = 0;
 		rq->prio_rotation = 0;
 		rq->active = rq->arrays;
+		rq->idleprio = rq->active;
 		rq->expired = rq->arrays + 1;
 		reset_prio_levels(rq);
 		rq->dyn_bitmap = rq->active->prio_bitmap;
@@ -6836,7 +6959,7 @@ void __init sched_init(void)
 		for (j = 0; j < 2; j++) {
 
 			array = rq->arrays + j;
-			for (k = 0; k < MAX_PRIO; k++)
+			for (k = 0; k <= MAX_PRIO; k++)
 				INIT_LIST_HEAD(array->queue + k);
 			bitmap_zero(array->prio_bitmap, MAX_PRIO);
 			/* delimiter for bitsearch */
1	Add the SCHED_IDLEPRIO scheduling policy. Tasks set to this policy are only
2	given cpu time if no other tasks at all wish to have cpu time thus running
3	effectively at idle priority. If semaphores or mutexes are held, or the
4	system is going into suspend, schedule them as SCHED_NORMAL nice 19.
5
6	Signed-off-by: Con Kolivas <kernel@kolivas.org>
7	---
8	include/linux/sched.h \| 5 +
9	kernel/sched.c \| 145 ++++++++++++++++++++++++++++++++++++++++++++++----
10	2 files changed, 138 insertions(+), 12 deletions(-)
11
12	Index: linux-2.6.21-ck2/include/linux/sched.h
13	===================================================================
14	--- linux-2.6.21-ck2.orig/include/linux/sched.h 2007-05-14 19:30:31.000000000 +1000
15	+++ linux-2.6.21-ck2/include/linux/sched.h 2007-05-14 19:30:31.000000000 +1000
16	@@ -35,10 +35,11 @@
17	#define SCHED_RR 2
18	#define SCHED_BATCH 3
19	#define SCHED_ISO 4
20	+#define SCHED_IDLEPRIO 5
21
22	#ifdef __KERNEL__
23
24	-#define SCHED_MAX SCHED_ISO
25	+#define SCHED_MAX SCHED_IDLEPRIO
26	#define SCHED_RANGE(policy) ((policy) <= SCHED_MAX)
27
28	struct sched_param {
29	@@ -537,6 +538,7 @@ struct signal_struct {
30	(policy) == SCHED_RR)
31	#define has_rt_policy(p) unlikely(is_rt_policy((p)->policy))
32	#define iso_task(p) unlikely((p)->policy == SCHED_ISO)
33	+#define idleprio_task(p) unlikely((p)->policy == SCHED_IDLEPRIO)
34
35	/*
36	* Some day this will be a full-fledged user tracking system..
37	@@ -1173,6 +1175,7 @@ static inline void put_task_struct(struc
38	#define PF_ISOREF 0x04000000 /* SCHED_ISO task has used up quota */
39	#define PF_MEMPOLICY 0x10000000 /* Non-default NUMA mempolicy */
40	#define PF_MUTEX_TESTER 0x20000000 /* Thread belongs to the rt mutex tester */
41	+#define PF_NONSLEEP 0x40000000 /* Waiting on in-kernel activity */
42
43	/*
44	* Only the _current_ task can read/write to tsk->flags, but other
45	Index: linux-2.6.21-ck2/kernel/sched.c
46	===================================================================
47	--- linux-2.6.21-ck2.orig/kernel/sched.c 2007-05-14 19:30:31.000000000 +1000
48	+++ linux-2.6.21-ck2/kernel/sched.c 2007-05-14 19:30:31.000000000 +1000
49	@@ -144,7 +144,7 @@ struct rq;
50	*/
51	struct prio_array {
52	/* Tasks queued at each priority */
53	- struct list_head queue[MAX_PRIO];
54	+ struct list_head queue[MAX_PRIO + 1];
55
56	/*
57	* The bitmap of priorities queued for this array. While the expired
58	@@ -201,7 +201,7 @@ struct rq {
59	unsigned long next_balance;
60	struct mm_struct *prev_mm;
61
62	- struct prio_array active, expired, arrays[2];
63	+ struct prio_array active, expired, *idleprio, arrays[2];
64	unsigned long dyn_bitmap, exp_bitmap;
65
66	/*
67	@@ -215,6 +215,8 @@ struct rq {
68	unsigned long iso_ticks;
69	unsigned short iso_refractory;
70
71	+ /* Number of idleprio tasks running */
72	+ unsigned long nr_idleprio;
73	atomic_t nr_iowait;
74
75	#ifdef CONFIG_SMP
76	@@ -652,6 +654,17 @@ sched_info_switch(struct task_struct *pr
77	#define sched_info_switch(t, next) do { } while (0)
78	#endif /* CONFIG_SCHEDSTATS \|\| CONFIG_TASK_DELAY_ACCT */
79
80	+static int idleprio_suitable(struct task_struct *p)
81	+{
82	+ return (!p->mutexes_held && !freezing(p) && !signal_pending(p) &&
83	+ !(p->flags & (PF_NONSLEEP \| PF_EXITING)));
84	+}
85	+
86	+static int idleprio(const struct task_struct *p)
87	+{
88	+ return (p->prio == MAX_PRIO);
89	+}
90	+
91	static inline int task_queued(struct task_struct *task)
92	{
93	return !list_empty(&task->run_list);
94	@@ -668,7 +681,9 @@ static inline void set_dynamic_bit(struc
95	static void dequeue_task(struct task_struct p, struct rq rq)
96	{
97	list_del_init(&p->run_list);
98	- if (list_empty(p->array->queue + p->prio))
99	+ if (idleprio_task(p) && idleprio(p))
100	+ rq->nr_idleprio--;
101	+ else if (list_empty(p->array->queue + p->prio))
102	__clear_bit(p->prio, p->array->prio_bitmap);
103	}
104
105	@@ -809,6 +824,8 @@ static inline int isoprio_suitable(struc
106	return !(p->flags & PF_ISOREF);
107	}
108
109	+static int task_timeslice(struct task_struct *p);
110	+
111	/*
112	* recalc_task_prio determines what priority a non rt_task will be
113	* queued at. If the task has already been running during this runqueue's
114	@@ -842,6 +859,30 @@ static void recalc_task_prio(struct task
115	/* Just about to be demoted to SCHED_NORMAL */
116	p->time_slice = 0;
117	}
118	+ } else if (idleprio_task(p)) {
119	+ if (idleprio_suitable(p)) {
120	+ /*
121	+ * If suitable idleprio_tasks are queued at MAX_PRIO
122	+ * only on the idleprio array. Their time_slice is
123	+ * their full task_timeslice as they cooperatively
124	+ * multitask.
125	+ */
126	+ p->prio = p->normal_prio = MAX_PRIO;
127	+ p->array = rq->idleprio;
128	+ if (p->time_slice <= 0)
129	+ p->time_slice = task_timeslice(p);
130	+ return;
131	+ }
132	+ /*
133	+ * If unsuitable idleprio_tasks are queued equivalent to
134	+ * nice 19 tasks on the expired array.
135	+ */
136	+ p->flags &= ~PF_NONSLEEP;
137	+ p->prio = p->normal_prio = MAX_PRIO - 1;
138	+ p->array = rq->expired;
139	+ if (p->time_slice <= 0 \|\| p->time_slice > p->quota)
140	+ p->time_slice = p->quota;
141	+ return;
142	}
143
144	update_if_moved(p, rq);
145	@@ -878,6 +919,8 @@ static inline void __enqueue_task(struct
146	else
147	recalc_task_prio(p, rq);
148
149	+ if (idleprio_task(p) && idleprio(p))
150	+ rq->nr_idleprio++;
151	sched_info_queued(p);
152	set_dynamic_bit(p, rq);
153	}
154	@@ -942,6 +985,8 @@ static int task_timeslice(struct task_st
155
156	static void set_load_weight(struct task_struct *p)
157	{
158	+ int load_weight;
159	+
160	if (has_rt_policy(p)) {
161	#ifdef CONFIG_SMP
162	if (p == task_rq(p)->migration_thread)
163	@@ -950,12 +995,19 @@ static void set_load_weight(struct task_
164	* Giving its load any weight will skew balancing
165	* adversely.
166	*/
167	- p->load_weight = 0;
168	+ load_weight = 0;
169	else
170	#endif
171	- p->load_weight = RTPRIO_TO_LOAD_WEIGHT(p->rt_priority);
172	+ load_weight = RTPRIO_TO_LOAD_WEIGHT(p->rt_priority);
173	} else
174	- p->load_weight = task_timeslice(p);
175	+ load_weight = task_timeslice(p);
176	+ /*
177	+ * idleprio tasks have much lower weight than SCHED_NORMAL tasks but
178	+ * still need to be weighted to allow balancing to occur.
179	+ */
180	+ if (likely(!idleprio_task(p)))
181	+ load_weight *= PRIO_RANGE;
182	+ p->load_weight = load_weight;
183	}
184
185	static inline void
186	@@ -1653,6 +1705,14 @@ out_activate:
187	out_running:
188	p->state = TASK_RUNNING;
189	out:
190	+ /*
191	+ * Special case when freezing we need to reschedule idleprio tasks
192	+ * as SCHED_NORMAL or else they'll never freeze
193	+ */
194	+ if (idleprio_task(p) && freezing(p) && idleprio(p)) {
195	+ dequeue_task(p, rq);
196	+ enqueue_task(p, rq);
197	+ }
198	task_rq_unlock(rq, &flags);
199
200	return success;
201	@@ -2263,7 +2323,9 @@ skip_bitmap:
202	idx = sched_find_first_bit(array->prio_bitmap);
203	else
204	idx = find_next_bit(array->prio_bitmap, MAX_PRIO, idx);
205	- if (idx >= MAX_PRIO) {
206	+ if (idx == MAX_PRIO) {
207	+ if (array == busiest->idleprio && busiest->nr_idleprio)
208	+ goto found_idleprio;
209	if (array == busiest->expired) {
210	array = busiest->active;
211	goto new_array;
212	@@ -2271,6 +2333,7 @@ skip_bitmap:
213	goto out;
214	}
215
216	+found_idleprio:
217	head = array->queue + idx;
218	curr = head->prev;
219	skip_queue:
220	@@ -2292,6 +2355,17 @@ skip_queue:
221	best_prio_seen \|= idx == best_prio;
222	if (curr != head)
223	goto skip_queue;
224	+ if (idx == MAX_PRIO) {
225	+ /*
226	+ * Occurs either when balancing idleprio tasks or
227	+ * there really are no more tasks to find.
228	+ */
229	+ if (array == busiest->expired) {
230	+ array = busiest->active;
231	+ goto new_array;
232	+ }
233	+ goto out;
234	+ }
235	idx++;
236	goto skip_bitmap;
237	}
238	@@ -2309,6 +2383,13 @@ skip_queue:
239	this_best_prio = idx;
240	if (curr != head)
241	goto skip_queue;
242	+ if (idx == MAX_PRIO) {
243	+ if (array == busiest->expired) {
244	+ array = busiest->active;
245	+ goto new_array;
246	+ }
247	+ goto out;
248	+ }
249	idx++;
250	goto skip_bitmap;
251	}
252	@@ -3136,7 +3217,7 @@ void account_user_time(struct task_struc
253
254	/* Add user time to cpustat. */
255	tmp = cputime_to_cputime64(cputime);
256	- if (TASK_NICE(p) > 0)
257	+ if (TASK_NICE(p) > 0 \|\| idleprio_task(p))
258	cpustat->nice = cputime64_add(cpustat->nice, tmp);
259	else
260	cpustat->user = cputime64_add(cpustat->user, tmp);
261	@@ -3351,6 +3432,28 @@ static void reset_prio_levels(struct rq
262	}
263
264	/*
265	+ * Only tasks running are SCHED_IDLEPRIO. Set the active array to the
266	+ * idleprio array and if it isn't already active
267	+ */
268	+static struct task_struct next_idleprio_task(struct rq rq)
269	+{
270	+ struct prio_array *array = rq->active;
271	+ struct list_head *queue;
272	+
273	+ if (array != rq->idleprio) {
274	+ rq->active = rq->idleprio;
275	+ rq->expired = array;
276	+ array = rq->active;
277	+ rq->exp_bitmap = rq->expired->prio_bitmap;
278	+ rq->dyn_bitmap = rq->active->prio_bitmap;
279	+ }
280	+ rq->prio_rotation++;
281	+ reset_prio_levels(rq);
282	+ queue = array->queue + MAX_PRIO;
283	+ return list_entry(queue->next, struct task_struct, run_list);
284	+}
285	+
286	+/*
287	* next_dynamic_task finds the next suitable dynamic task.
288	*/
289	static inline struct task_struct next_dynamic_task(struct rq rq, int idx)
290	@@ -3361,6 +3464,8 @@ static inline struct task_struct *next_d
291	int nstatic;
292
293	retry:
294	+ if (unlikely(rq->nr_running == rq->nr_idleprio))
295	+ return next_idleprio_task(rq);
296	if (idx >= MAX_PRIO) {
297	/* There are no more tasks in the active array. Swap arrays */
298	array = rq->expired;
299	@@ -3451,8 +3556,10 @@ need_resched_nonpreemptible:
300	unlikely(signal_pending(prev))))
301	prev->state = TASK_RUNNING;
302	else {
303	- if (prev->state == TASK_UNINTERRUPTIBLE)
304	+ if (prev->state == TASK_UNINTERRUPTIBLE) {
305	+ prev->flags \|= PF_NONSLEEP;
306	rq->nr_uninterruptible++;
307	+ }
308	deactivate_task(prev, rq);
309	}
310	}
311	@@ -3994,7 +4101,8 @@ void set_user_nice(struct task_struct *p
312	* If the task increased its priority or is running and
313	* lowered its priority, then reschedule its CPU:
314	*/
315	- if (delta < 0 \|\| (delta > 0 && task_running(rq, p)))
316	+ if (delta < 0 \|\| ((delta > 0 \|\| idleprio_task(p)) &&
317	+ task_running(rq, p)))
318	resched_task(rq->curr);
319	}
320	out_unlock:
321	@@ -4195,6 +4303,11 @@ recheck:
322	return -EPERM;
323	}
324
325	+ if (!(p->mm) && policy == SCHED_IDLEPRIO) {
326	+ /* Don't allow kernel threads to be SCHED_IDLEPRIO. */
327	+ return -EINVAL;
328	+ }
329	+
330	retval = security_task_setscheduler(p, policy, param);
331	if (retval)
332	return retval;
333	@@ -4520,12 +4633,18 @@ asmlinkage long sys_sched_yield(void)
334	struct prio_array *old_array = p->array;
335	int old_prio = p->prio;
336
337	+ if (idleprio_task(p)) {
338	+ dequeue_task(p, rq);
339	+ enqueue_task(p, rq);
340	+ goto out_release;
341	+ }
342	/* p->prio will be updated in requeue_task via queue_expired */
343	if (!rt_task(p))
344	p->array = rq->expired;
345	requeue_task(p, rq, old_array, old_prio);
346	}
347
348	+out_release:
349	/*
350	* Since we are going to call schedule() anyway, there's
351	* no need to preempt or enable interrupts:
352	@@ -4678,6 +4797,7 @@ asmlinkage long sys_sched_get_priority_m
353	case SCHED_NORMAL:
354	case SCHED_BATCH:
355	case SCHED_ISO:
356	+ case SCHED_IDLEPRIO:
357	ret = 0;
358	break;
359	}
360	@@ -4703,6 +4823,7 @@ asmlinkage long sys_sched_get_priority_m
361	case SCHED_NORMAL:
362	case SCHED_BATCH:
363	case SCHED_ISO:
364	+ case SCHED_IDLEPRIO:
365	ret = 0;
366	}
367	return ret;
368	@@ -6812,8 +6933,10 @@ void __init sched_init(void)
369	lockdep_set_class(&rq->lock, &rq->rq_lock_key);
370	rq->iso_ticks = 0;
371	rq->nr_running = 0;
372	+ rq->nr_idleprio = 0;
373	rq->prio_rotation = 0;
374	rq->active = rq->arrays;
375	+ rq->idleprio = rq->active;
376	rq->expired = rq->arrays + 1;
377	reset_prio_levels(rq);
378	rq->dyn_bitmap = rq->active->prio_bitmap;
379	@@ -6836,7 +6959,7 @@ void __init sched_init(void)
380	for (j = 0; j < 2; j++) {
381
382	array = rq->arrays + j;
383	- for (k = 0; k < MAX_PRIO; k++)
384	+ for (k = 0; k <= MAX_PRIO; k++)
385	INIT_LIST_HEAD(array->queue + k);
386	bitmap_zero(array->prio_bitmap, MAX_PRIO);
387	/* delimiter for bitsearch */