From 916cee2293928f1cab5b1739a121497afd14d804 Mon Sep 17 00:00:00 2001 From: "www.anuprita804@gmail.com" Date: Fri, 18 Aug 2017 16:31:10 +0200 Subject: [PATCH 01/86] sched: core: revert all oneplus changes --- kernel/sched/core.c | 108 ++++++-------------------------------------- 1 file changed, 15 insertions(+), 93 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d9e35807eaeb3..b1f006c3ad6c6 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -259,12 +259,14 @@ struct static_key sched_feat_keys[__SCHED_FEAT_NR] = { static void sched_feat_disable(int i) { - static_key_disable(&sched_feat_keys[i]); + if (static_key_enabled(&sched_feat_keys[i])) + static_key_slow_dec(&sched_feat_keys[i]); } static void sched_feat_enable(int i) { - static_key_enable(&sched_feat_keys[i]); + if (!static_key_enabled(&sched_feat_keys[i])) + static_key_slow_inc(&sched_feat_keys[i]); } #else static void sched_feat_disable(int i) { }; @@ -1686,21 +1688,6 @@ static void update_history(struct rq *rq, struct task_struct *p, if (!runtime || is_idle_task(p) || exiting_task(p) || !samples) goto done; -#ifdef VENDOR_EDIT - if (p->ravg.mitigated) { - /* update history accroding to previous records - * e.g. 60% loading in big cluster becomes 60% in smallest one - */ - for (ridx = 0; ridx < sched_ravg_hist_size; ridx++) { - hist[ridx] = div64_u64((u64)hist[ridx] * 1024 - , (u64)max_load_scale_factor); - sum += hist[ridx]; - if (hist[ridx] > max) - max = hist[ridx]; - } - goto update_demand; - } -#endif /* Push new 'runtime' value onto stack */ widx = sched_ravg_hist_size - 1; ridx = widx - samples; @@ -1718,9 +1705,6 @@ static void update_history(struct rq *rq, struct task_struct *p, max = hist[widx]; } -#ifdef VENDOR_EDIT -update_demand: -#endif p->ravg.sum = 0; if (p->on_rq) p->sched_class->dec_hmp_sched_stats(rq, p); @@ -1813,11 +1797,7 @@ static void update_task_demand(struct task_struct *p, struct rq *rq, u32 window_size = sched_ravg_window; new_window = mark_start < window_start; -#ifdef VENDOR_EDIT - if (!account_busy_for_task_demand(p, event) || p->ravg.mitigated) { -#else if (!account_busy_for_task_demand(p, event)) { -#endif if (new_window) /* If the time accounted isn't being accounted as * busy time, and a new window started, only the @@ -1826,13 +1806,6 @@ static void update_task_demand(struct task_struct *p, struct rq *rq, * elapsed, but since empty windows are dropped, * it is not necessary to account those. */ update_history(rq, p, p->ravg.sum, 1, event); -#ifdef VENDOR_EDIT - if (p->ravg.mitigated) { - /* force update history */ - update_history(rq, p, 1, RAVG_HIST_SIZE_MAX, event); - p->ravg.mitigated = 0; - } -#endif return; } @@ -4053,10 +4026,13 @@ static long calc_load_fold_active(struct rq *this_rq) static unsigned long calc_load(unsigned long load, unsigned long exp, unsigned long active) { - load *= exp; - load += active * (FIXED_1 - exp); - load += 1UL << (FSHIFT - 1); - return load >> FSHIFT; + unsigned long newload; + + newload = load * exp + active * (FIXED_1 - exp); + if (active >= load) + newload += FIXED_1-1; + + return newload / FIXED_1; } #ifdef CONFIG_NO_HZ_COMMON @@ -6836,8 +6812,7 @@ void show_state_filter(unsigned long state_filter) touch_all_softlockup_watchdogs(); #ifdef CONFIG_SYSRQ_SCHED_DEBUG - if (!state_filter) - sysrq_sched_debug_show(); + sysrq_sched_debug_show(); #endif rcu_read_unlock(); /* @@ -10087,11 +10062,6 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) if (period > max_cfs_quota_period) return -EINVAL; - /* - * Prevent race between setting of cfs_rq->runtime_enabled and - * unthrottle_offline_cfs_rqs(). - */ - get_online_cpus(); mutex_lock(&cfs_constraints_mutex); ret = __cfs_schedulable(tg, period, quota); if (ret) @@ -10113,11 +10083,12 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) /* restart the period timer (if active) to handle new period expiry */ if (runtime_enabled && cfs_b->timer_active) { /* force a reprogram */ - __start_cfs_bandwidth(cfs_b, true); + cfs_b->timer_active = 0; + __start_cfs_bandwidth(cfs_b); } raw_spin_unlock_irq(&cfs_b->lock); - for_each_online_cpu(i) { + for_each_possible_cpu(i) { struct cfs_rq *cfs_rq = tg->cfs_rq[i]; struct rq *rq = cfs_rq->rq; @@ -10133,39 +10104,10 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) cfs_bandwidth_usage_dec(); out_unlock: mutex_unlock(&cfs_constraints_mutex); - put_online_cpus(); return ret; } -#ifdef VENDOR_EDIT -int tg_set_cfs_quota_per_task(struct task_group *tg, long cfs_quota_us) -{ - struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; - u64 quota, period; - - period = ktime_to_ns(tg->cfs_bandwidth.period); - quota = tg->cfs_bandwidth.quota; - - raw_spin_lock_irq(&cfs_b->lock); - if (cfs_quota_us > 0) - cfs_b->quota_per_task = (u64)cfs_quota_us * NSEC_PER_USEC; - raw_spin_unlock_irq(&cfs_b->lock); - - return tg_set_cfs_bandwidth(tg, period, quota); -} - -long tg_get_cfs_quota_per_task(struct task_group *tg) -{ - u64 quota_us; - - quota_us = tg->cfs_bandwidth.quota_per_task; - do_div(quota_us, NSEC_PER_USEC); - - return quota_us; -} -#endif - int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) { u64 quota, period; @@ -10212,19 +10154,6 @@ long tg_get_cfs_period(struct task_group *tg) return cfs_period_us; } -#ifdef VENDOR_EDIT -static s64 cpu_cfs_quota_per_task_read_s64(struct cgroup *cgrp, struct cftype *cft) -{ - return tg_get_cfs_quota_per_task(cgroup_tg(cgrp)); -} - -static int cpu_cfs_quota_per_task_write_s64(struct cgroup *cgrp, struct cftype *cftype, - s64 cfs_quota_us) -{ - return tg_set_cfs_quota_per_task(cgroup_tg(cgrp), cfs_quota_us); -} -#endif - static s64 cpu_cfs_quota_read_s64(struct cgroup *cgrp, struct cftype *cft) { return tg_get_cfs_quota(cgroup_tg(cgrp)); @@ -10385,13 +10314,6 @@ static struct cftype cpu_files[] = { }, #endif #ifdef CONFIG_CFS_BANDWIDTH -#ifdef VENDOR_EDIT - { - .name = "cfs_quota_us_per_task", - .read_s64 = cpu_cfs_quota_per_task_read_s64, - .write_s64 = cpu_cfs_quota_per_task_write_s64, - }, -#endif { .name = "cfs_quota_us", .read_s64 = cpu_cfs_quota_read_s64, From f300f9d455adcf408a6dbf8a4cb384875fa36df8 Mon Sep 17 00:00:00 2001 From: "www.anuprita804@gmail.com" Date: Fri, 18 Aug 2017 16:41:32 +0200 Subject: [PATCH 02/86] Revert "fs: readdir: Fix su hide patch for non-iterate filesystems" This reverts commit 34295dc30935c2f9941e4b18bba514bb6e66295f. --- fs/readdir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/readdir.c b/fs/readdir.c index b22bbbfabeeb5..d52d18d98877a 100644 --- a/fs/readdir.c +++ b/fs/readdir.c @@ -37,9 +37,9 @@ int iterate_dir(struct file *file, struct dir_context *ctx) res = -ENOENT; if (!IS_DEADDIR(inode)) { - ctx->romnt = (inode->i_sb->s_flags & MS_RDONLY); if (file->f_op->iterate) { ctx->pos = file->f_pos; + ctx->romnt = (inode->i_sb->s_flags & MS_RDONLY); res = file->f_op->iterate(file, ctx); file->f_pos = ctx->pos; } else { From ace1aebfe32485acf2402500bf4d1a027dec2901 Mon Sep 17 00:00:00 2001 From: "www.anuprita804@gmail.com" Date: Fri, 18 Aug 2017 16:43:09 +0200 Subject: [PATCH 03/86] Revert "kernel: Fix potential refcount leak in su check" This reverts commit 20ad8a951b24ec89e887070364b386169b3ad89a. --- fs/namei.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/namei.c b/fs/namei.c index bdfbb34dd137d..40d9a30eaf3ac 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -2029,10 +2029,8 @@ static int path_lookupat(int dfd, const char *name, if (!err) { struct super_block *sb = nd->inode->i_sb; if (sb->s_flags & MS_RDONLY) { - if (d_is_su(nd->path.dentry) && !su_visible()) { - path_put(&nd->path); + if (d_is_su(nd->path.dentry) && !su_visible()) err = -ENOENT; - } } } From e69f34b663ccfba6bdd6c1b4137b228340b1666e Mon Sep 17 00:00:00 2001 From: "www.anuprita804@gmail.com" Date: Fri, 18 Aug 2017 16:43:33 +0200 Subject: [PATCH 04/86] Revert "kernel: Only expose su when daemon is running" This reverts commit bd8e647de61e69cd0516526c169280a9489a731e. --- fs/exec.c | 5 ----- fs/namei.c | 8 -------- fs/readdir.c | 15 --------------- include/linux/dcache.h | 7 ------- include/linux/fs.h | 1 - include/linux/sched.h | 8 -------- include/linux/uidgid.h | 4 ---- kernel/exit.c | 4 ---- kernel/fork.c | 2 -- kernel/sched/core.c | 32 -------------------------------- 10 files changed, 86 deletions(-) diff --git a/fs/exec.c b/fs/exec.c index 0ca8cbac20cd4..618d3ac29b368 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1570,11 +1570,6 @@ static int do_execve_common(const char *filename, if (retval < 0) goto out; - if (d_is_su(file->f_dentry) && capable(CAP_SYS_ADMIN)) { - current->flags |= PF_SU; - su_exec(); - } - /* execve succeeded */ current->fs->in_exec = 0; current->in_execve = 0; diff --git a/fs/namei.c b/fs/namei.c index 40d9a30eaf3ac..06942ce4f7e75 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -2026,14 +2026,6 @@ static int path_lookupat(int dfd, const char *name, } } - if (!err) { - struct super_block *sb = nd->inode->i_sb; - if (sb->s_flags & MS_RDONLY) { - if (d_is_su(nd->path.dentry) && !su_visible()) - err = -ENOENT; - } - } - if (base) fput(base); diff --git a/fs/readdir.c b/fs/readdir.c index d52d18d98877a..d46eca8567a41 100644 --- a/fs/readdir.c +++ b/fs/readdir.c @@ -39,7 +39,6 @@ int iterate_dir(struct file *file, struct dir_context *ctx) if (!IS_DEADDIR(inode)) { if (file->f_op->iterate) { ctx->pos = file->f_pos; - ctx->romnt = (inode->i_sb->s_flags & MS_RDONLY); res = file->f_op->iterate(file, ctx); file->f_pos = ctx->pos; } else { @@ -54,14 +53,6 @@ int iterate_dir(struct file *file, struct dir_context *ctx) } EXPORT_SYMBOL(iterate_dir); -static bool hide_name(const char *name, int namlen) -{ - if (namlen == 2 && !memcmp(name, "su", 2)) - if (!su_visible()) - return true; - return false; -} - /* * Traditional linux readdir() handling.. * @@ -100,8 +91,6 @@ static int fillonedir(void * __buf, const char * name, int namlen, loff_t offset buf->result = -EOVERFLOW; return -EOVERFLOW; } - if (hide_name(name, namlen) && buf->ctx.romnt) - return 0; buf->result++; dirent = buf->dirent; if (!access_ok(VERIFY_WRITE, dirent, @@ -179,8 +168,6 @@ static int filldir(void * __buf, const char * name, int namlen, loff_t offset, buf->error = -EOVERFLOW; return -EOVERFLOW; } - if (hide_name(name, namlen) && buf->ctx.romnt) - return 0; dirent = buf->previous; if (dirent) { if (__put_user(offset, &dirent->d_off)) @@ -259,8 +246,6 @@ static int filldir64(void * __buf, const char * name, int namlen, loff_t offset, buf->error = -EINVAL; /* only used if we fail.. */ if (reclen > buf->count) return -EINVAL; - if (hide_name(name, namlen) && buf->ctx.romnt) - return 0; dirent = buf->previous; if (dirent) { if (__put_user(offset, &dirent->d_off)) diff --git a/include/linux/dcache.h b/include/linux/dcache.h index f2c042f1cb81a..e91a63f63e462 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h @@ -413,13 +413,6 @@ static inline bool d_mountpoint(struct dentry *dentry) return dentry->d_flags & DCACHE_MOUNTED; } -static inline bool d_is_su(const struct dentry *dentry) -{ - return dentry && - dentry->d_name.len == 2 && - !memcmp(dentry->d_name.name, "su", 2); -} - extern int sysctl_vfs_cache_pressure; #endif /* __LINUX_DCACHE_H */ diff --git a/include/linux/fs.h b/include/linux/fs.h index 4682822c3fc91..9632f0bb43b6e 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1535,7 +1535,6 @@ typedef int (*filldir_t)(void *, const char *, int, loff_t, u64, unsigned); struct dir_context { const filldir_t actor; loff_t pos; - bool romnt; }; static inline bool dir_emit(struct dir_context *ctx, diff --git a/include/linux/sched.h b/include/linux/sched.h index 53b9e2d0da438..784674e554fda 100755 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -55,12 +55,6 @@ struct sched_param { #include -int su_instances(void); -bool su_running(void); -bool su_visible(void); -void su_exec(void); -void su_exit(void); - #define SCHED_ATTR_SIZE_VER0 48 /* sizeof first published struct */ /* @@ -1806,8 +1800,6 @@ static inline void sched_set_io_is_busy(int val) {}; #define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */ #define PF_WAKE_UP_IDLE 0x80000000 /* try to wake up on an idle CPU */ -#define PF_SU 0x00000002 /* task is su */ - /* * Only the _current_ task can read/write to tsk->flags, but other * tasks can access tsk->flags in readonly mode for example diff --git a/include/linux/uidgid.h b/include/linux/uidgid.h index 5fc3c7f3bcc73..e7a8f1332f893 100755 --- a/include/linux/uidgid.h +++ b/include/linux/uidgid.h @@ -67,10 +67,6 @@ static inline gid_t __kgid_val(kgid_t gid) //huruihuan add for cgroup control #define GLOBAL_SYSTEM_UID KUIDT_INIT(1000) #endif - -#define GLOBAL_SYSTEM_UID KUIDT_INIT(1000) -#define GLOBAL_SYSTEM_GID KGIDT_INIT(1000) - #define INVALID_UID KUIDT_INIT(-1) #define INVALID_GID KGIDT_INIT(-1) diff --git a/kernel/exit.c b/kernel/exit.c index e58c525cbc3ab..540bad43812df 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -777,10 +777,6 @@ void do_exit(long code) sched_exit(tsk); - if (tsk->flags & PF_SU) { - su_exit(); - } - /* * tsk->flags are checked in the futex code to protect against * an exiting task cleaning up the robust pi futexes. diff --git a/kernel/fork.c b/kernel/fork.c index 4adf3bb29d2a1..78d02637079c1 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -326,8 +326,6 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) if (err) goto free_ti; - tsk->flags &= ~PF_SU; - tsk->stack = ti; #ifdef CONFIG_SECCOMP /* diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b1f006c3ad6c6..0c9de80b14b8e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -114,38 +114,6 @@ do { \ local_irq_restore(dflags); \ } while (0) -static atomic_t __su_instances; - -int su_instances(void) -{ - return atomic_read(&__su_instances); -} - -bool su_running(void) -{ - return su_instances() > 0; -} - -bool su_visible(void) -{ - kuid_t uid = current_uid(); - if (su_running()) - return true; - if (uid_eq(uid, GLOBAL_ROOT_UID) || uid_eq(uid, GLOBAL_SYSTEM_UID)) - return true; - return false; -} - -void su_exec(void) -{ - atomic_inc(&__su_instances); -} - -void su_exit(void) -{ - atomic_dec(&__su_instances); -} - const char *task_event_names[] = {"PUT_PREV_TASK", "PICK_NEXT_TASK", "TASK_WAKE", "TASK_MIGRATE", "TASK_UPDATE", "IRQ_UPDATE"}; From bd8dab85224fabdff6eee5814c653a367703aba2 Mon Sep 17 00:00:00 2001 From: Joe Maples Date: Thu, 27 Jul 2017 18:19:04 -0500 Subject: [PATCH 05/86] Revert "sched: prevent task migration while governor queries CPUs' load" This reverts commit 14573fa10a3456356d62e96392b5626ab1c92c83. Signed-off-by: Joe Maples --- include/linux/sched.h | 2 - kernel/sched/core.c | 94 +++++++++++++------------------------------ 2 files changed, 29 insertions(+), 67 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 784674e554fda..53add51bfba40 100755 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1749,8 +1749,6 @@ extern int task_free_unregister(struct notifier_block *n); #ifdef CONFIG_SCHED_FREQ_INPUT extern int sched_set_window(u64 window_start, unsigned int window_size); extern unsigned long sched_get_busy(int cpu); -extern void sched_get_cpus_busy(unsigned long *busy, - const struct cpumask *query_cpus); extern void sched_set_io_is_busy(int val); #else static inline int sched_set_window(u64 window_start, unsigned int window_size) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0c9de80b14b8e..1dd1f82633310 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2133,87 +2133,51 @@ scale_load_to_freq(u64 load, unsigned int src_freq, unsigned int dst_freq) return div64_u64(load * (u64)src_freq, (u64)dst_freq); } -void sched_get_cpus_busy(unsigned long *busy, const struct cpumask *query_cpus) +unsigned long sched_get_busy(int cpu) { unsigned long flags; - struct rq *rq; - const int cpus = cpumask_weight(query_cpus); - u64 load[cpus]; - unsigned int cur_freq[cpus], max_freq[cpus]; - int notifier_sent[cpus]; - int cpu, i = 0; - unsigned int window_size; - - if (unlikely(cpus == 0)) - return; + struct rq *rq = cpu_rq(cpu); + u64 load; /* * This function could be called in timer context, and the * current task may have been executing for a long time. Ensure * that the window stats are current by doing an update. */ - local_irq_save(flags); - for_each_cpu(cpu, query_cpus) - raw_spin_lock(&cpu_rq(cpu)->lock); - - window_size = sched_ravg_window; - - for_each_cpu(cpu, query_cpus) { - rq = cpu_rq(cpu); + raw_spin_lock_irqsave(&rq->lock, flags); + update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), 0); + load = rq->old_busy_time = rq->prev_runnable_sum; - update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), 0); - load[i] = rq->old_busy_time = rq->prev_runnable_sum; - /* - * Scale load in reference to rq->max_possible_freq. - * - * Note that scale_load_to_cpu() scales load in reference to - * rq->max_freq. - */ - load[i] = scale_load_to_cpu(load[i], cpu); + /* + * Scale load in reference to rq->max_possible_freq. + * + * Note that scale_load_to_cpu() scales load in reference to + * rq->max_freq + */ + load = scale_load_to_cpu(load, cpu); - notifier_sent[i] = rq->notifier_sent; + if (!rq->notifier_sent) { + u64 load_at_cur_freq; + + load_at_cur_freq = scale_load_to_freq(load, rq->max_freq, + rq->cur_freq); + if (load_at_cur_freq > sched_ravg_window) + load_at_cur_freq = sched_ravg_window; + load = scale_load_to_freq(load_at_cur_freq, + rq->cur_freq, rq->max_possible_freq); + } else { + load = scale_load_to_freq(load, rq->max_freq, + rq->max_possible_freq); rq->notifier_sent = 0; - cur_freq[i] = rq->cur_freq; - max_freq[i] = rq->max_freq; - i++; } - for_each_cpu(cpu, query_cpus) - raw_spin_unlock(&(cpu_rq(cpu))->lock); - local_irq_restore(flags); - - i = 0; - for_each_cpu(cpu, query_cpus) { - rq = cpu_rq(cpu); - - if (!notifier_sent[i]) { - load[i] = scale_load_to_freq(load[i], max_freq[i], - cur_freq[i]); - if (load[i] > window_size) - load[i] = window_size; - load[i] = scale_load_to_freq(load[i], cur_freq[i], - rq->max_possible_freq); - } else { - load[i] = scale_load_to_freq(load[i], max_freq[i], - rq->max_possible_freq); - } - - busy[i] = div64_u64(load[i], NSEC_PER_USEC); - - trace_sched_get_busy(cpu, busy[i]); - i++; - } -} + load = div64_u64(load, NSEC_PER_USEC); -unsigned long sched_get_busy(int cpu) -{ - struct cpumask query_cpu = CPU_MASK_NONE; - unsigned long busy; + raw_spin_unlock_irqrestore(&rq->lock, flags); - cpumask_set_cpu(cpu, &query_cpu); - sched_get_cpus_busy(&busy, &query_cpu); + trace_sched_get_busy(cpu, load); - return busy; + return load; } void sched_set_io_is_busy(int val) From 88bf466c474b77feae801dec3658b786f9399353 Mon Sep 17 00:00:00 2001 From: Joe Maples Date: Thu, 27 Jul 2017 18:21:28 -0500 Subject: [PATCH 06/86] Revert "sched: Add stub function for sched_get_cpus_busy()" This reverts commit bccfaf5d2df6084e8e55a6d4519759970701e219. Signed-off-by: Joe Maples --- include/linux/sched.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 53add51bfba40..a662980a50741 100755 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1759,8 +1759,6 @@ static inline unsigned long sched_get_busy(int cpu) { return 0; } -static inline void sched_get_cpus_busy(unsigned long *busy, - const struct cpumask *query_cpus) {}; static inline void sched_set_io_is_busy(int val) {}; #endif From 943295684251bb29bab31fd35847776791aa9cf0 Mon Sep 17 00:00:00 2001 From: anupritaisno1 Date: Thu, 27 Jul 2017 18:27:02 -0500 Subject: [PATCH 07/86] eas: staging for op2 --- arch/arm64/Kconfig | 6 --- kernel/sched/core.c | 116 ++----------------------------------------- kernel/sched/sched.h | 14 ------ 3 files changed, 3 insertions(+), 133 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 61cc7178e2b28..1d0f32bb4f1b6 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -407,12 +407,6 @@ config SCHED_SMT MultiThreading at a cost of slightly increased overhead in some places. If unsure say N here. -config ARCH_WANTS_CTXSW_LOGGING - bool "Enable logging in context switch" - depends on SMP - help - Logs critical context switch latencies. If unsure say N here. - config NR_CPUS int "Maximum number of CPUs (2-32)" range 2 32 diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 1dd1f82633310..c522004a3069d 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -90,30 +90,6 @@ #define CREATE_TRACE_POINTS #include -#include - -#define DLOG_SIZE 15000 -#define MAX_CTXSW_LATENCY 1000000000 -static DEFINE_PER_CPU(char[DLOG_SIZE], dbuf); -static DEFINE_PER_CPU(char *, dptr); - -#define dlog(x...) \ -do { \ - unsigned long dflags; \ - char *ptr, *buf; \ - int dcpu; \ - local_irq_save(dflags); \ - dcpu = smp_processor_id(); \ - buf = per_cpu(dbuf, dcpu); \ - ptr = per_cpu(dptr, dcpu); \ - ptr += snprintf(ptr, 10, "CPU %d ", dcpu); \ - ptr += snprintf(ptr, 490, x); \ - if (ptr - buf > DLOG_SIZE - 500) \ - ptr = buf; \ - per_cpu(dptr, dcpu) = ptr; \ - local_irq_restore(dflags); \ -} while (0) - const char *task_event_names[] = {"PUT_PREV_TASK", "PICK_NEXT_TASK", "TASK_WAKE", "TASK_MIGRATE", "TASK_UPDATE", "IRQ_UPDATE"}; @@ -1328,8 +1304,6 @@ static inline unsigned int load_to_freq(struct rq *rq, u64 load) static int send_notification(struct rq *rq) { unsigned int cur_freq, freq_required; - unsigned long flags; - int rc = 0; if (!sched_enable_hmp) return 0; @@ -1340,14 +1314,7 @@ static int send_notification(struct rq *rq) if (nearly_same_freq(cur_freq, freq_required)) return 0; - raw_spin_lock_irqsave(&rq->lock, flags); - if (!rq->notifier_sent) { - rq->notifier_sent = 1; - rc = 1; - } - raw_spin_unlock_irqrestore(&rq->lock, flags); - - return rc; + return 1; } /* Alert governor if there is a need to change frequency */ @@ -2127,12 +2094,6 @@ void reset_all_window_stats(u64 window_start, unsigned int window_size) #ifdef CONFIG_SCHED_FREQ_INPUT -static inline u64 -scale_load_to_freq(u64 load, unsigned int src_freq, unsigned int dst_freq) -{ - return div64_u64(load * (u64)src_freq, (u64)dst_freq); -} - unsigned long sched_get_busy(int cpu) { unsigned long flags; @@ -2147,6 +2108,7 @@ unsigned long sched_get_busy(int cpu) raw_spin_lock_irqsave(&rq->lock, flags); update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), 0); load = rq->old_busy_time = rq->prev_runnable_sum; + raw_spin_unlock_irqrestore(&rq->lock, flags); /* * Scale load in reference to rq->max_possible_freq. @@ -2155,26 +2117,9 @@ unsigned long sched_get_busy(int cpu) * rq->max_freq */ load = scale_load_to_cpu(load, cpu); - - if (!rq->notifier_sent) { - u64 load_at_cur_freq; - - load_at_cur_freq = scale_load_to_freq(load, rq->max_freq, - rq->cur_freq); - if (load_at_cur_freq > sched_ravg_window) - load_at_cur_freq = sched_ravg_window; - load = scale_load_to_freq(load_at_cur_freq, - rq->cur_freq, rq->max_possible_freq); - } else { - load = scale_load_to_freq(load, rq->max_freq, - rq->max_possible_freq); - rq->notifier_sent = 0; - } - + load = div64_u64(load * (u64)rq->max_freq, (u64)rq->max_possible_freq); load = div64_u64(load, NSEC_PER_USEC); - raw_spin_unlock_irqrestore(&rq->lock, flags); - trace_sched_get_busy(cpu, load); return load; @@ -2319,10 +2264,8 @@ static void __update_min_max_capacity(void) static void update_min_max_capacity(void) { - unsigned long flags; int i; - local_irq_save(flags); for_each_possible_cpu(i) raw_spin_lock(&cpu_rq(i)->lock); @@ -2330,7 +2273,6 @@ static void update_min_max_capacity(void) for_each_possible_cpu(i) raw_spin_unlock(&cpu_rq(i)->lock); - local_irq_restore(flags); } /* @@ -3202,25 +3144,6 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) goto stat; #ifdef CONFIG_SMP - /* - * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be - * possible to, falsely, observe p->on_cpu == 0. - * - * One must be running (->on_cpu == 1) in order to remove oneself - * from the runqueue. - * - * [S] ->on_cpu = 1; [L] ->on_rq - * UNLOCK rq->lock - * RMB - * LOCK rq->lock - * [S] ->on_rq = 0; [L] ->on_cpu - * - * Pairs with the full barrier implied in the UNLOCK+LOCK on rq->lock - * from the consecutive calls to schedule(); the first switching to our - * task, the second putting it to sleep. - */ - smp_rmb(); - /* * If the owning (remote) cpu is still in the middle of schedule() with * this task as prev, wait until its done referencing the task. @@ -3620,9 +3543,6 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next) { trace_sched_switch(prev, next); -#ifdef CONFIG_ARCH_WANTS_CTXSW_LOGGING - dlog("%s: end trace at %llu\n", __func__, sched_clock()); -#endif sched_info_switch(prev, next); perf_event_task_sched_out(prev, next); fire_sched_out_preempt_notifiers(prev, next); @@ -3756,19 +3676,11 @@ context_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next) { struct mm_struct *mm, *oldmm; -#ifdef CONFIG_ARCH_WANTS_CTXSW_LOGGING - u64 start, end; -#endif prepare_task_switch(rq, prev, next); mm = next->mm; oldmm = prev->active_mm; - -#ifdef CONFIG_ARCH_WANTS_CTXSW_LOGGING - start = sched_clock(); - dlog("%s: n_mm: %p, finish pts at %llu\n", __func__, mm, start); -#endif /* * For paravirt, this is coupled with an exit in switch_to to * combine the page table reload and the switch backend into @@ -3787,15 +3699,6 @@ context_switch(struct rq *rq, struct task_struct *prev, prev->active_mm = NULL; rq->prev_mm = oldmm; } - -#ifdef CONFIG_ARCH_WANTS_CTXSW_LOGGING - end = sched_clock(); - dlog("%s: start task switch at %llu\n", __func__, end); - - if (end - start > MAX_CTXSW_LATENCY) - msm_trigger_wdog_bite(); -#endif - /* * Since the runqueue lock will be released by the next * task (which is an invalid locking op but in the case @@ -4778,10 +4681,6 @@ static void __sched __schedule(void) smp_mb__before_spinlock(); raw_spin_lock_irq(&rq->lock); -#ifdef CONFIG_ARCH_WANTS_CTXSW_LOGGING - dlog("%s: locked %p at %llu\n", __func__, &rq->lock, sched_clock()); -#endif - switch_count = &prev->nivcsw; if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { if (unlikely(signal_pending_state(prev->state, prev))) { @@ -4826,10 +4725,6 @@ static void __sched __schedule(void) rq->curr = next; ++*switch_count; -#ifdef CONFIG_ARCH_WANTS_CTXSW_LOGGING - dlog("%s: enter context_switch at %llu\n", - __func__, sched_clock()); -#endif context_switch(rq, prev, next); /* unlocks the rq */ /* * The context switch have flipped the stack from under us @@ -5894,7 +5789,6 @@ int sched_setscheduler_nocheck(struct task_struct *p, int policy, }; return __sched_setscheduler(p, &attr, false); } -EXPORT_SYMBOL(sched_setscheduler_nocheck); static int do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) @@ -7281,7 +7175,6 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) set_window_start(rq); raw_spin_unlock_irqrestore(&rq->lock, flags); rq->calc_load_update = calc_load_update; - account_reset_rq(rq); break; case CPU_ONLINE: @@ -9067,7 +8960,6 @@ void __init sched_init(void) for_each_possible_cpu(i) { struct rq *rq; - per_cpu(dptr, i) = per_cpu(dbuf, i); rq = cpu_rq(i); raw_spin_lock_init(&rq->lock); rq->nr_running = 0; @@ -9129,7 +9021,6 @@ void __init sched_init(void) rq->wakeup_latency = 0; rq->wakeup_energy = 0; #ifdef CONFIG_SCHED_HMP - cpumask_set_cpu(i, &rq->freq_domain_cpumask); rq->cur_freq = 1; rq->max_freq = 1; rq->min_freq = 1; @@ -9152,7 +9043,6 @@ void __init sched_init(void) #ifdef CONFIG_SCHED_FREQ_INPUT rq->old_busy_time = 0; rq->curr_runnable_sum = rq->prev_runnable_sum = 0; - rq->notifier_sent = 0; #endif #endif diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index bde547591ed3e..a47b87294572d 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -542,7 +542,6 @@ struct rq { #ifdef CONFIG_SCHED_FREQ_INPUT unsigned int old_busy_time; - int notifier_sent; #endif #endif @@ -1730,16 +1729,3 @@ static inline u64 irq_time_read(int cpu) } #endif /* CONFIG_64BIT */ #endif /* CONFIG_IRQ_TIME_ACCOUNTING */ - -static inline void account_reset_rq(struct rq *rq) -{ -#ifdef CONFIG_IRQ_TIME_ACCOUNTING - rq->prev_irq_time = 0; -#endif -#ifdef CONFIG_PARAVIRT - rq->prev_steal_time = 0; -#endif -#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING - rq->prev_steal_time_rq = 0; -#endif -} From 1a8b65a16bfe8aea8d6afe77eeddd5d5473db67b Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Mon, 14 Dec 2015 14:35:14 -0800 Subject: [PATCH 08/86] Add EAS CONFIGs Change-Id: I900b9a93d320fc335b1a7b5bab32f466223beef1 # Conflicts: # arch/arm64/configs/oneplus2_defconfig --- arch/arm64/configs/oneplus2_defconfig | 8 +++-- init/Kconfig | 45 +++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/arch/arm64/configs/oneplus2_defconfig b/arch/arm64/configs/oneplus2_defconfig index 6ad8080730f8a..dadd570e4dce8 100644 --- a/arch/arm64/configs/oneplus2_defconfig +++ b/arch/arm64/configs/oneplus2_defconfig @@ -1,5 +1,9 @@ CONFIG_LOCALVERSION="-perf" CONFIG_SYSVIPC=y +CONFIG_SCHED_TUNE=y +CONFIG_CGROUP_SCHEDTUNE=y +CONFIG_CPU_FREQ_GOV_SCHED=y +CONFIG_RT_MUTEXES=y CONFIG_AUDIT=y CONFIG_NO_HZ=y CONFIG_HIGH_RES_TIMERS=y @@ -12,9 +16,9 @@ CONFIG_CPUSETS=y CONFIG_CGROUP_CPUACCT=y CONFIG_RESOURCE_COUNTERS=y CONFIG_CGROUP_SCHED=y -CONFIG_CFS_BANDWIDTH=y +CONFIG_CFS_BANDWIDTH=n CONFIG_RT_GROUP_SCHED=y -CONFIG_SCHED_HMP=y +CONFIG_SCHED_HMP=n CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_IPC_NS is not set diff --git a/init/Kconfig b/init/Kconfig index c353169a708fe..8fba3ca0bd3b6 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -867,6 +867,24 @@ config RESOURCE_COUNTERS This option enables controller independent resource accounting infrastructure that works with cgroups. +config CGROUP_SCHEDTUNE + bool "Task boosting cgroup subsystem for EAS (EXPERIMENTAL)" + depends on SCHED_TUNE + help + This option provides the "schedtune" controller which improve the + flexibility of the task boosting mechanism by introducing the support + to define "per task" boost values. + + This new controller: + 1. allows only a two layers hierarchy, where the root defines the + system-wide boost value and its direct childs define each one a + different "class of tasks" to be boosted with a different value + 2. supports up to 16 different task classes, each one which could be + configured with a different boost value + + Only if you are testing a kernel with energy-aware scheduler + support, you might want to say Y here. + config MEMCG bool "Memory Resource Controller for Control Groups" depends on RESOURCE_COUNTERS @@ -1143,6 +1161,33 @@ config SCHED_AUTOGROUP desktop applications. Task group autogeneration is currently based upon task session. +config SCHED_TUNE + bool "Tasks boosting for energy-aware scheduler (EXPERIMENTAL)" + help + This option enable the system-wide support for task boosting. + When this support is enabled a new sysctl interface is exposed to + userspace via: + /proc/sys/kernel/sched_cfs_boost + which allows to set a system-wide boost value in range [0..100]. + + The currently boosting strategy is implemented in such a way that: + - a 0% boost value requires to operate in "standard" EAS mode by + scheduling all tasks at the minimum capacities required by their + workload demand + - a 100% boost value requires to push at maximum the task + performances, "regardless" of the incurred energy consumption + + A boost value in between these two boundaries is used to bias the + power/performance trade-off, the higher the boost value the more the + EAS scheduler is biased toward performance boosting instead of energy + efficiency. + + Since this support exposes a single system-wide knob, the specified + boost value is applied to all (CFS) tasks in the system. + + Only if you are testing a kernel with energy-aware scheduler support, + you might want to say Y here. + config MM_OWNER bool From d80e8a57ad0c9d04da5edf4a579e36cecab020d2 Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Mon, 14 Dec 2015 15:39:42 -0800 Subject: [PATCH 09/86] Add EAS topology changes Change-Id: If7da362a7d13085cf318879de86f350481436d4b --- arch/arm64/include/asm/topology.h | 4 + arch/arm64/kernel/topology.c | 88 ++++++++++++++++----- include/linux/sched.h | 22 +++++- include/linux/sched_energy.h | 36 +++++++++ include/linux/smp.h | 1 + kernel/sched/Makefile | 2 +- kernel/sched/core.c | 7 +- kernel/sched/energy.c | 124 ++++++++++++++++++++++++++++++ kernel/sched/fair.c | 50 ++++++------ kernel/smp.c | 99 ++++++++++++++++++++++++ 10 files changed, 383 insertions(+), 50 deletions(-) create mode 100644 include/linux/sched_energy.h create mode 100644 kernel/sched/energy.c diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index 7ebcd31ce51ca..abf8c9a8c9378 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -24,6 +24,10 @@ void init_cpu_topology(void); void store_cpu_topology(unsigned int cpuid); const struct cpumask *cpu_coregroup_mask(int cpu); +struct sched_domain; +extern unsigned long arch_scale_freq_capacity(struct sched_domain *sd, int cpu); +DECLARE_PER_CPU(atomic_long_t, cpu_freq_capacity); + #else static inline void init_cpu_topology(void) { } diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 75e586914d8b2..245825c0fc2b1 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -38,14 +39,14 @@ */ static DEFINE_PER_CPU(unsigned long, cpu_scale); -unsigned long arch_scale_freq_power(struct sched_domain *sd, int cpu) +unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu) { return per_cpu(cpu_scale, cpu); } -static void set_power_scale(unsigned int cpu, unsigned long power) +static void set_capacity_scale(unsigned int cpu, unsigned long capacity) { - per_cpu(cpu_scale, cpu) = power; + per_cpu(cpu_scale, cpu) = capacity; } static int __init get_cpu_for_node(struct device_node *node) @@ -195,13 +196,10 @@ struct cpu_efficiency { * Table of relative efficiency of each processors * The efficiency value must fit in 20bit and the final * cpu_scale value must be in the range - * 0 < cpu_scale < 3*SCHED_POWER_SCALE/2 - * in order to return at most 1 when DIV_ROUND_CLOSEST - * is used to compute the capacity of a CPU. - * Processors that are not defined in the table, - * use the default SCHED_POWER_SCALE value for cpu_scale. + * 0 < cpu_scale < SCHED_CAPACITY_SCALE */ static const struct cpu_efficiency table_efficiency[] = { + { "arm,cortex-a72", 4186 }, { "arm,cortex-a57", 3891 }, { "arm,cortex-a53", 2048 }, { NULL, }, @@ -212,12 +210,14 @@ static unsigned long *__cpu_capacity; static unsigned long middle_capacity = 1; -static DEFINE_PER_CPU(unsigned long, cpu_efficiency) = SCHED_POWER_SCALE; +#ifdef TJK_HMP +static DEFINE_PER_CPU(unsigned long, cpu_efficiency) = SCHED_CAPACITY_SCALE; unsigned long arch_get_cpu_efficiency(int cpu) { return per_cpu(cpu_efficiency, cpu); } +#endif /* * Iterate all CPUs' descriptor in DT and compute the efficiency @@ -270,7 +270,24 @@ static int __init parse_dt_topology(void) return ret; } -static void __init parse_dt_cpu_power(void) +/* + * Scheduler load-tracking scale-invariance + * + * Provides the scheduler with a scale-invariance correction factor that + * compensates for frequency scaling (arch_scale_freq_capacity()). The scaling + * factor is updated in smp.c + */ +unsigned long arch_scale_freq_capacity(struct sched_domain *sd, int cpu) +{ + unsigned long curr = atomic_long_read(&per_cpu(cpu_freq_capacity, cpu)); + + if (!curr) + return SCHED_CAPACITY_SCALE; + + return curr; +} + +static void __init parse_dt_cpu_capacity(void) { const struct cpu_efficiency *cpu_eff; struct device_node *cn; @@ -302,7 +319,9 @@ static void __init parse_dt_cpu_power(void) continue; } +#ifdef TJK_HMP per_cpu(cpu_efficiency, cpu) = cpu_eff->efficiency; +#endif rate = of_get_property(cn, "clock-frequency", &len); if (!rate || len != 4) { @@ -328,19 +347,20 @@ static void __init parse_dt_cpu_power(void) * cpu_scale because all CPUs have the same capacity. Otherwise, we * compute a middle_capacity factor that will ensure that the capacity * of an 'average' CPU of the system will be as close as possible to - * SCHED_POWER_SCALE, which is the default value, but with the + * SCHED_CAPACITY_SCALE, which is the default value, but with the * constraint explained near table_efficiency[]. */ if (min_capacity == max_capacity) return; else if (4 * max_capacity < (3 * (max_capacity + min_capacity))) middle_capacity = (min_capacity + max_capacity) - >> (SCHED_POWER_SHIFT+1); + >> (SCHED_CAPACITY_SHIFT+1); else middle_capacity = ((max_capacity / 3) - >> (SCHED_POWER_SHIFT-1)) + 1; + >> (SCHED_CAPACITY_SHIFT-1)) + 1; } +#ifdef TJK_HMP /* * Look for a customed capacity of a CPU in the cpu_topo_data table during the * boot. The update of all CPUs is in O(n^2) for heteregeneous system but the @@ -354,8 +374,9 @@ static void update_cpu_power(unsigned int cpu) set_power_scale(cpu, cpu_capacity(cpu) / middle_capacity); pr_info("CPU%u: update cpu_power %lu\n", - cpu, arch_scale_freq_power(NULL, cpu)); + cpu, arch_scale_cpu_capacity(NULL, cpu)); } +#endif /* * cpu topology table @@ -363,11 +384,39 @@ static void update_cpu_power(unsigned int cpu) struct cpu_topology cpu_topology[NR_CPUS]; EXPORT_SYMBOL_GPL(cpu_topology); +static inline const struct sched_group_energy *cpu_core_energy(int cpu) +{ + struct sched_group_energy *sge = sge_array[cpu][SD_LEVEL0]; + + if (!sge) { + pr_warn("Invalid sched_group_energy for CPU%d\n", cpu); + return NULL; + } + + return sge; +} + const struct cpumask *cpu_coregroup_mask(int cpu) { return &cpu_topology[cpu].core_sibling; } +static void update_cpu_capacity(unsigned int cpu) +{ + unsigned long capacity; + + if (!cpu_core_energy(cpu)) { + capacity = SCHED_CAPACITY_SCALE; + } else { + int max_cap_idx = cpu_core_energy(cpu)->nr_cap_states - 1; + capacity = cpu_core_energy(cpu)->cap_states[max_cap_idx].cap; + } + + set_capacity_scale(cpu, capacity); + pr_info("CPU%d: update cpu_capacity %lu\n", + cpu, arch_scale_cpu_capacity(NULL, cpu)); +} + static void update_siblings_masks(unsigned int cpuid) { struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid]; @@ -440,7 +489,7 @@ void store_cpu_topology(unsigned int cpuid) topology_populated: update_siblings_masks(cpuid); - update_cpu_power(cpuid); + update_cpu_capacity(cpuid); } static void __init reset_cpu_topology(void) @@ -461,12 +510,12 @@ static void __init reset_cpu_topology(void) } } -static void __init reset_cpu_power(void) +static void __init reset_cpu_capacity(void) { unsigned int cpu; for_each_possible_cpu(cpu) - set_power_scale(cpu, SCHED_POWER_SCALE); + set_capacity_scale(cpu, SCHED_CAPACITY_SCALE); } void __init init_cpu_topology(void) @@ -480,6 +529,7 @@ void __init init_cpu_topology(void) if (parse_dt_topology()) reset_cpu_topology(); - reset_cpu_power(); - parse_dt_cpu_power(); + reset_cpu_capacity(); + parse_dt_cpu_capacity(); + init_sched_energy_costs(); } diff --git a/include/linux/sched.h b/include/linux/sched.h index a662980a50741..a048d0b46d413 100755 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -837,8 +837,8 @@ enum cpu_idle_type { /* * Increase resolution of cpu_power calculations */ -#define SCHED_POWER_SHIFT 10 -#define SCHED_POWER_SCALE (1L << SCHED_POWER_SHIFT) +#define SCHED_CAPACITY_SHIFT 10 +#define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT) /* * sched-domains (multiprocessor balancing) declarations: @@ -869,6 +869,24 @@ struct sched_domain_attr { extern int sched_domain_level_max; +struct capacity_state { + unsigned long cap; /* compute capacity */ + unsigned long power; /* power consumption at this compute capacity */ +}; + +struct idle_state { + unsigned long power; /* power consumption in this idle state */ +}; + +struct sched_group_energy { + atomic_t ref; + unsigned int nr_idle_states; /* number of idle states */ + struct idle_state *idle_states; /* ptr to idle state array */ + unsigned int nr_idle_states_below; /* number idle states in lower groups */ + unsigned int nr_cap_states; /* number of capacity states */ + struct capacity_state *cap_states; /* ptr to capacity state array */ +}; + struct sched_group; struct sched_domain { diff --git a/include/linux/sched_energy.h b/include/linux/sched_energy.h new file mode 100644 index 0000000000000..a3f1627ac609e --- /dev/null +++ b/include/linux/sched_energy.h @@ -0,0 +1,36 @@ +#ifndef _LINUX_SCHED_ENERGY_H +#define _LINUX_SCHED_ENERGY_H + +#include +#include + +/* + * There doesn't seem to be an NR_CPUS style max number of sched domain + * levels so here's an arbitrary constant one for the moment. + * + * The levels alluded to here correspond to entries in struct + * sched_domain_topology_level that are meant to be populated by arch + * specific code (topology.c). + */ +#define NR_SD_LEVELS 8 + +#define SD_LEVEL0 0 +#define SD_LEVEL1 1 +#define SD_LEVEL2 2 +#define SD_LEVEL3 3 +#define SD_LEVEL4 4 +#define SD_LEVEL5 5 +#define SD_LEVEL6 6 +#define SD_LEVEL7 7 + +/* + * Convenience macro for iterating through said sd levels. + */ +#define for_each_possible_sd_level(level) \ + for (level = 0; level < NR_SD_LEVELS; level++) + +extern struct sched_group_energy *sge_array[NR_CPUS][NR_SD_LEVELS]; + +void init_sched_energy_costs(void); + +#endif diff --git a/include/linux/smp.h b/include/linux/smp.h index 868dcf0b2eb1a..c7d3d55419f2c 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -28,6 +28,7 @@ extern unsigned int total_cpus; int smp_call_function_single(int cpuid, smp_call_func_t func, void *info, int wait); +int smp_call_function_single_async(int cpuid, struct call_single_data *csd); #ifdef CONFIG_SMP diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 1c7bde63c5e5b..0f4e25cc07ac7 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -11,7 +11,7 @@ ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer endif -obj-y += core.o clock.o cputime.o idle_task.o fair.o rt.o stop_task.o sched_avg.o +obj-y += core.o clock.o cputime.o idle_task.o fair.o rt.o stop_task.o sched_avg.o energy.o obj-$(CONFIG_SMP) += cpupri.o obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o obj-$(CONFIG_SCHEDSTATS) += stats.o diff --git a/kernel/sched/core.c b/kernel/sched/core.c index c522004a3069d..78550c6187ed0 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7359,7 +7359,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group)); printk(KERN_CONT " %s", str); - if (group->sgp->power != SCHED_POWER_SCALE) { + if (group->sgp->power != SCHED_CAPACITY_SCALE) { printk(KERN_CONT " (cpu_power = %d)", group->sgp->power); } @@ -7826,7 +7826,7 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu) * domains and no possible iteration will get us here, we won't * die on a /0 trap. */ - sg->sgp->power = SCHED_POWER_SCALE * cpumask_weight(sg_span); + sg->sgp->power = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span); /* * Make sure the first group of this domain contains the @@ -9007,7 +9007,7 @@ void __init sched_init(void) #ifdef CONFIG_SMP rq->sd = NULL; rq->rd = NULL; - rq->cpu_power = SCHED_POWER_SCALE; + rq->cpu_power = SCHED_CAPACITY_SCALE; rq->post_schedule = 0; rq->active_balance = 0; rq->next_balance = jiffies; @@ -10188,3 +10188,4 @@ void dump_cpu_task(int cpu) pr_info("Task dump for CPU %d:\n", cpu); sched_show_task(cpu_curr(cpu)); } + diff --git a/kernel/sched/energy.c b/kernel/sched/energy.c new file mode 100644 index 0000000000000..b0656b7a93e33 --- /dev/null +++ b/kernel/sched/energy.c @@ -0,0 +1,124 @@ +/* + * Obtain energy cost data from DT and populate relevant scheduler data + * structures. + * + * Copyright (C) 2015 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#define pr_fmt(fmt) "sched-energy: " fmt + +#define DEBUG + +#include +#include +#include +#include +#include +#include + +struct sched_group_energy *sge_array[NR_CPUS][NR_SD_LEVELS]; + +static void free_resources(void) +{ + int cpu, sd_level; + struct sched_group_energy *sge; + + for_each_possible_cpu(cpu) { + for_each_possible_sd_level(sd_level) { + sge = sge_array[cpu][sd_level]; + if (sge) { + kfree(sge->cap_states); + kfree(sge->idle_states); + kfree(sge); + } + } + } +} + +void init_sched_energy_costs(void) +{ + struct device_node *cn, *cp; + struct capacity_state *cap_states; + struct idle_state *idle_states; + struct sched_group_energy *sge; + const struct property *prop; + int sd_level, i, nstates, cpu; + const __be32 *val; + + for_each_possible_cpu(cpu) { + cn = of_get_cpu_node(cpu, NULL); + if (!cn) { + pr_warn("CPU device node missing for CPU %d\n", cpu); + return; + } + + if (!of_find_property(cn, "sched-energy-costs", NULL)) { + pr_warn("CPU device node has no sched-energy-costs\n"); + return; + } + + for_each_possible_sd_level(sd_level) { + cp = of_parse_phandle(cn, "sched-energy-costs", sd_level); + if (!cp) + break; + + prop = of_find_property(cp, "busy-cost-data", NULL); + if (!prop || !prop->value) { + pr_warn("No busy-cost data, skipping sched_energy init\n"); + goto out; + } + + sge = kcalloc(1, sizeof(struct sched_group_energy), + GFP_NOWAIT); + + nstates = (prop->length / sizeof(u32)) / 2; + cap_states = kcalloc(nstates, + sizeof(struct capacity_state), + GFP_NOWAIT); + + for (i = 0, val = prop->value; i < nstates; i++) { + cap_states[i].cap = be32_to_cpup(val++); + cap_states[i].power = be32_to_cpup(val++); + } + + sge->nr_cap_states = nstates; + sge->cap_states = cap_states; + + prop = of_find_property(cp, "idle-cost-data", NULL); + if (!prop || !prop->value) { + pr_warn("No idle-cost data, skipping sched_energy init\n"); + goto out; + } + + nstates = (prop->length / sizeof(u32)); + idle_states = kcalloc(nstates, + sizeof(struct idle_state), + GFP_NOWAIT); + + for (i = 0, val = prop->value; i < nstates; i++) + idle_states[i].power = be32_to_cpup(val++); + + sge->nr_idle_states = nstates; + sge->idle_states = idle_states; + + sge_array[cpu][sd_level] = sge; + } + } + + pr_info("Sched-energy-costs installed from DT\n"); + return; + +out: + free_resources(); +} diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 60d836e5ecb88..8942198ac1b85 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2965,7 +2965,7 @@ static inline int find_new_hmp_ilb(int call_cpu, int type) static inline int power_cost(u64 task_load, int cpu) { - return SCHED_POWER_SCALE; + return SCHED_CAPACITY_SCALE; } static inline int @@ -5532,7 +5532,7 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, } /* Adjust by relative CPU power of the group */ - avg_load = (avg_load * SCHED_POWER_SCALE) / group->sgp->power; + avg_load = (avg_load * SCHED_CAPACITY_SCALE) / group->sgp->power; if (local_group) { this_load = avg_load; @@ -6681,7 +6681,7 @@ static inline int get_sd_load_idx(struct sched_domain *sd, static unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu) { - return SCHED_POWER_SCALE; + return SCHED_CAPACITY_SCALE; } unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu) @@ -6725,10 +6725,10 @@ static unsigned long scale_rt_power(int cpu) available = total - avg; } - if (unlikely((s64)total < SCHED_POWER_SCALE)) - total = SCHED_POWER_SCALE; + if (unlikely((s64)total < SCHED_CAPACITY_SCALE)) + total = SCHED_CAPACITY_SCALE; - total >>= SCHED_POWER_SHIFT; + total >>= SCHED_CAPACITY_SHIFT; return div_u64(available, total); } @@ -6736,7 +6736,7 @@ static unsigned long scale_rt_power(int cpu) static void update_cpu_power(struct sched_domain *sd, int cpu) { unsigned long weight = sd->span_weight; - unsigned long power = SCHED_POWER_SCALE; + unsigned long power = SCHED_CAPACITY_SCALE; struct sched_group *sdg = sd->groups; if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) { @@ -6745,7 +6745,7 @@ static void update_cpu_power(struct sched_domain *sd, int cpu) else power *= default_scale_smt_power(sd, cpu); - power >>= SCHED_POWER_SHIFT; + power >>= SCHED_CAPACITY_SHIFT; } sdg->sgp->power_orig = power; @@ -6755,10 +6755,10 @@ static void update_cpu_power(struct sched_domain *sd, int cpu) else power *= default_scale_freq_power(sd, cpu); - power >>= SCHED_POWER_SHIFT; + power >>= SCHED_CAPACITY_SHIFT; power *= scale_rt_power(cpu); - power >>= SCHED_POWER_SHIFT; + power >>= SCHED_CAPACITY_SHIFT; if (!power) power = 1; @@ -6820,7 +6820,7 @@ static inline int fix_small_capacity(struct sched_domain *sd, struct sched_group *group) { /* - * Only siblings can have significantly less than SCHED_POWER_SCALE + * Only siblings can have significantly less than SCHED_CAPACITY_SCALE */ if (!(sd->flags & SD_SHARE_CPUPOWER)) return 0; @@ -6924,7 +6924,7 @@ static inline void update_sg_lb_stats(struct lb_env *env, } /* Adjust by relative CPU power of the group */ - sgs->avg_load = (sgs->group_load*SCHED_POWER_SCALE) / group->sgp->power; + sgs->avg_load = (sgs->group_load*SCHED_CAPACITY_SCALE) / group->sgp->power; /* * Consider the group unbalanced when the imbalance is larger @@ -6943,7 +6943,7 @@ static inline void update_sg_lb_stats(struct lb_env *env, sgs->group_imb = 1; sgs->group_capacity = DIV_ROUND_CLOSEST(group->sgp->power, - SCHED_POWER_SCALE); + SCHED_CAPACITY_SCALE); if (!sgs->group_capacity) sgs->group_capacity = fix_small_capacity(env->sd, group); sgs->group_weight = group->group_weight; @@ -7177,7 +7177,7 @@ static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds) return 0; env->imbalance = DIV_ROUND_CLOSEST( - sds->max_load * sds->busiest->sgp->power, SCHED_POWER_SCALE); + sds->max_load * sds->busiest->sgp->power, SCHED_CAPACITY_SCALE); return 1; } @@ -7207,7 +7207,7 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds) } scaled_busy_load_per_task = sds->busiest_load_per_task - * SCHED_POWER_SCALE; + * SCHED_CAPACITY_SCALE; scaled_busy_load_per_task /= sds->busiest->sgp->power; if (sds->max_load - sds->this_load + scaled_busy_load_per_task >= @@ -7226,10 +7226,10 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds) min(sds->busiest_load_per_task, sds->max_load); pwr_now += sds->this->sgp->power * min(sds->this_load_per_task, sds->this_load); - pwr_now /= SCHED_POWER_SCALE; + pwr_now /= SCHED_CAPACITY_SCALE; /* Amount of load we'd subtract */ - tmp = (sds->busiest_load_per_task * SCHED_POWER_SCALE) / + tmp = (sds->busiest_load_per_task * SCHED_CAPACITY_SCALE) / sds->busiest->sgp->power; if (sds->max_load > tmp) pwr_move += sds->busiest->sgp->power * @@ -7237,15 +7237,15 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds) /* Amount of load we'd add */ if (sds->max_load * sds->busiest->sgp->power < - sds->busiest_load_per_task * SCHED_POWER_SCALE) + sds->busiest_load_per_task * SCHED_CAPACITY_SCALE) tmp = (sds->max_load * sds->busiest->sgp->power) / sds->this->sgp->power; else - tmp = (sds->busiest_load_per_task * SCHED_POWER_SCALE) / + tmp = (sds->busiest_load_per_task * SCHED_CAPACITY_SCALE) / sds->this->sgp->power; pwr_move += sds->this->sgp->power * min(sds->this_load_per_task, sds->this_load + tmp); - pwr_move /= SCHED_POWER_SCALE; + pwr_move /= SCHED_CAPACITY_SCALE; /* Move if we gain throughput */ if (pwr_move > pwr_now) @@ -7285,7 +7285,7 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s load_above_capacity = (sds->busiest_nr_running - sds->busiest_group_capacity); - load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_POWER_SCALE); + load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_CAPACITY_SCALE); load_above_capacity /= sds->busiest->sgp->power; } @@ -7305,7 +7305,7 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s /* How much load to actually move to equalise the imbalance */ env->imbalance = min(max_pull * sds->busiest->sgp->power, (sds->avg_load - sds->this_load) * sds->this->sgp->power) - / SCHED_POWER_SCALE; + / SCHED_CAPACITY_SCALE; /* * if *imbalance is less than the average load per runnable task @@ -7373,7 +7373,7 @@ find_busiest_group(struct lb_env *env, int *balance) if (bail_inter_cluster_balance(env, &sds)) goto out_balanced; - sds.avg_load = (SCHED_POWER_SCALE * sds.total_load) / sds.total_pwr; + sds.avg_load = (SCHED_CAPACITY_SCALE * sds.total_load) / sds.total_pwr; /* * If the busiest group is imbalanced the below checks don't @@ -7499,7 +7499,7 @@ static struct rq *find_busiest_queue(struct lb_env *env, for_each_cpu(i, sched_group_cpus(group)) { unsigned long power = power_of(i); unsigned long capacity = DIV_ROUND_CLOSEST(power, - SCHED_POWER_SCALE); + SCHED_CAPACITY_SCALE); unsigned long wl; if (!capacity) @@ -7524,7 +7524,7 @@ static struct rq *find_busiest_queue(struct lb_env *env, * the load can be moved away from the cpu that is potentially * running at a lower capacity. */ - wl = (wl * SCHED_POWER_SCALE) / power; + wl = (wl * SCHED_CAPACITY_SCALE) / power; if (wl > max_load) { max_load = wl; diff --git a/kernel/smp.c b/kernel/smp.c index dec409c83ddba..a835ecba883d2 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include "smpboot.h" @@ -273,6 +275,18 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info, } EXPORT_SYMBOL(smp_call_function_single); +int smp_call_function_single_async(int cpu, struct call_single_data *csd) +{ + int err = 0; + + preempt_disable(); + generic_exec_single(cpu, csd, 0); + preempt_enable(); + + return err; +} +EXPORT_SYMBOL_GPL(smp_call_function_single_async); + /* * smp_call_function_any - Run a function on any of the given cpus * @mask: The mask of cpus it can run on. @@ -719,3 +733,88 @@ void kick_all_cpus_sync(void) smp_call_function(do_nothing, NULL, 1); } EXPORT_SYMBOL_GPL(kick_all_cpus_sync); + + + +#ifdef CONFIG_CPU_FREQ + +static DEFINE_PER_CPU(atomic_long_t, cpu_max_freq); +DEFINE_PER_CPU(atomic_long_t, cpu_freq_capacity); + +/* + * Scheduler load-tracking scale-invariance + * + * Provides the scheduler with a scale-invariance correction factor that + * compensates for frequency scaling through arch_scale_freq_capacity() + * (implemented in topology.c). + */ +static inline +void scale_freq_capacity(int cpu, unsigned long curr, unsigned long max) +{ + unsigned long capacity; + + if (!max) + return; + + capacity = (curr << SCHED_CAPACITY_SHIFT) / max; + atomic_long_set(&per_cpu(cpu_freq_capacity, cpu), capacity); +} + +static int cpufreq_callback(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_freqs *freq = data; + int cpu = freq->cpu; + unsigned long max = atomic_long_read(&per_cpu(cpu_max_freq, cpu)); + + if (freq->flags & CPUFREQ_CONST_LOOPS) + return NOTIFY_OK; + + if (val == CPUFREQ_PRECHANGE) { + scale_freq_capacity(cpu, freq->new, max); + // TJK: fix tracing: trace_cpu_capacity(capacity_curr_of(cpu), cpu); + } + + return NOTIFY_OK; +} + +static struct notifier_block cpufreq_notifier = { + .notifier_call = cpufreq_callback, +}; + +static int cpufreq_policy_callback(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_policy *policy = data; + int i; + + if (val != CPUFREQ_NOTIFY) + return NOTIFY_OK; + + for_each_cpu(i, policy->cpus) { + scale_freq_capacity(i, policy->cur, policy->max); + atomic_long_set(&per_cpu(cpu_max_freq, i), policy->max); + } + + return NOTIFY_OK; +} + +static struct notifier_block cpufreq_policy_notifier = { + .notifier_call = cpufreq_policy_callback, +}; + +static int __init register_cpufreq_notifier(void) +{ + int ret; + + ret = cpufreq_register_notifier(&cpufreq_notifier, + CPUFREQ_TRANSITION_NOTIFIER); + if (ret) + return ret; + + return cpufreq_register_notifier(&cpufreq_policy_notifier, + CPUFREQ_POLICY_NOTIFIER); +} +core_initcall(register_cpufreq_notifier); +#endif + From 94f79bc63e01a7d05a7411756579415ab5a629a6 Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Mon, 14 Dec 2015 15:51:18 -0800 Subject: [PATCH 10/86] Add changes to non-sched files for EAS Change-Id: I38be889e84b0610285aeb231cb031e422279ee7d --- drivers/cpufreq/Kconfig | 24 ++++++++++++++++++++++++ drivers/cpufreq/cpufreq_interactive.c | 5 +++++ fs/proc/base.c | 9 ++++++++- include/linux/cputime.h | 16 ++++++++++++++++ include/linux/preempt.h | 9 ++++++++- include/linux/sched.h | 2 +- include/trace/events/power.h | 6 ++++++ include/uapi/linux/sched.h | 2 ++ kernel/fork.c | 2 +- kernel/sched/core.c | 8 ++++---- kernel/sysctl.c | 6 ++++++ 11 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 include/linux/cputime.h diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index 29afa37aa4c12..2b2030fe895c7 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -118,6 +118,15 @@ config CPU_FREQ_DEFAULT_GOV_INTERACTIVE loading your cpufreq low-level hardware driver, using the 'interactive' governor for latency-sensitive workloads. +config CPU_FREQ_DEFAULT_GOV_SCHED + bool "sched" + select CPU_FREQ_GOV_SCHED + select CPU_FREQ_GOV_PERFORMANCE + help + Use the CPUfreq governor 'sched' as default. This scales + cpu frequency from the scheduler as per-entity load tracking + statistics are updated. + endchoice config CPU_FREQ_GOV_PERFORMANCE @@ -216,6 +225,21 @@ config CPU_FREQ_GOV_CONSERVATIVE If in doubt, say N. +config CPU_FREQ_GOV_SCHED + tristate "'sched' cpufreq governor" + depends on CPU_FREQ + select CPU_FREQ_GOV_COMMON + help + 'sched' - this governor scales cpu frequency from the + scheduler as a function of cpu capacity utilization. It does + not evaluate utilization on a periodic basis (as ondemand + does) but instead is invoked from the completely fair + scheduler when updating per-entity load tracking statistics. + Latency to respond to changes in load is improved over polling + governors due to its event-driven design. + + If in doubt, say N. + config CPU_BOOST tristate "Event base short term CPU freq boost" depends on CPU_FREQ diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c index 59990ac20f22c..ba888b776d775 100644 --- a/drivers/cpufreq/cpufreq_interactive.c +++ b/drivers/cpufreq/cpufreq_interactive.c @@ -36,6 +36,11 @@ #define CREATE_TRACE_POINTS #include +#ifndef TJK_HMP +ATOMIC_NOTIFIER_HEAD(migration_notifier_head); +ATOMIC_NOTIFIER_HEAD(load_alert_notifier_head); +#endif + struct cpufreq_interactive_policyinfo { struct timer_list policy_timer; struct timer_list policy_slack_timer; diff --git a/fs/proc/base.c b/fs/proc/base.c index d3410433ed49b..b5d1fedf486bb 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -1326,11 +1326,14 @@ static int sched_wake_up_idle_show(struct seq_file *m, void *v) if (!p) return -ESRCH; +#ifdef TJK_HMP seq_printf(m, "%d\n", sched_get_wake_up_idle(p)); put_task_struct(p); - return 0; +#else + return -EINVAL; +#endif } static ssize_t @@ -1358,9 +1361,13 @@ sched_wake_up_idle_write(struct file *file, const char __user *buf, if (!p) return -ESRCH; +#ifdef TJK_HMP err = sched_set_wake_up_idle(p, wake_up_idle); put_task_struct(p); +#else + err = -EINVAL; +#endif out: return err < 0 ? err : count; diff --git a/include/linux/cputime.h b/include/linux/cputime.h new file mode 100644 index 0000000000000..f2eb2ee535cab --- /dev/null +++ b/include/linux/cputime.h @@ -0,0 +1,16 @@ +#ifndef __LINUX_CPUTIME_H +#define __LINUX_CPUTIME_H + +#include + +#ifndef cputime_to_nsecs +# define cputime_to_nsecs(__ct) \ + (cputime_to_usecs(__ct) * NSEC_PER_USEC) +#endif + +#ifndef nsecs_to_cputime +# define nsecs_to_cputime(__nsecs) \ + usecs_to_cputime((__nsecs) / NSEC_PER_USEC) +#endif + +#endif /* __LINUX_CPUTIME_H */ diff --git a/include/linux/preempt.h b/include/linux/preempt.h index f5d4723cdb3d3..9a66079428676 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -10,7 +10,7 @@ #include #include -#if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER) +#if 1 && (defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER)) extern void add_preempt_count(int val); extern void sub_preempt_count(int val); #else @@ -23,6 +23,10 @@ #define preempt_count() (current_thread_info()->preempt_count) +#ifndef TJK_HMP +#define set_preempt_need_resched() +#endif + #ifdef CONFIG_PREEMPT asmlinkage void preempt_schedule(void); @@ -126,6 +130,9 @@ do { \ #endif /* CONFIG_PREEMPT_COUNT */ +#define preempt_set_need_resched() +#define preempt_fold_need_resched() + #ifdef CONFIG_PREEMPT_NOTIFIERS struct preempt_notifier; diff --git a/include/linux/sched.h b/include/linux/sched.h index a048d0b46d413..70ad9d64cb031 100755 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2210,7 +2210,7 @@ extern void wake_up_new_task(struct task_struct *tsk); #else static inline void kick_process(struct task_struct *tsk) { } #endif -extern void sched_fork(struct task_struct *p); +extern void sched_fork(unsigned long clone_flags, struct task_struct *p); extern void sched_dead(struct task_struct *p); #ifdef CONFIG_SCHED_HMP extern void sched_exit(struct task_struct *p); diff --git a/include/trace/events/power.h b/include/trace/events/power.h index 65fe4ea8e3e01..47c34d93a85f1 100644 --- a/include/trace/events/power.h +++ b/include/trace/events/power.h @@ -539,6 +539,12 @@ DEFINE_EVENT(cpu_modes, cpu_mode_detect, mode, cpu_cnt) ); +DEFINE_EVENT(cpu, cpu_capacity, + + TP_PROTO(unsigned int capacity, unsigned int cpu_id), + TP_ARGS(capacity, cpu_id) +); + DECLARE_EVENT_CLASS(timer_status, TP_PROTO(unsigned int cpu, unsigned int single_enter_cycles, unsigned int single_enter_cycle_cnt, diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h index 5a0f945927ace..cf1a839091117 100644 --- a/include/uapi/linux/sched.h +++ b/include/uapi/linux/sched.h @@ -39,8 +39,10 @@ #define SCHED_BATCH 3 /* SCHED_ISO: reserved but not implemented yet */ #define SCHED_IDLE 5 +#define SCHED_DEADLINE 6 /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ #define SCHED_RESET_ON_FORK 0x40000000 +#define SCHED_FLAG_RESET_ON_FORK 0x01 #endif /* _UAPI_LINUX_SCHED_H */ diff --git a/kernel/fork.c b/kernel/fork.c index 78d02637079c1..a743abbfb69da 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1373,7 +1373,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, #endif /* Perform scheduler related setup. Assign this task to a CPU. */ - sched_fork(p); + sched_fork(clone_flags, p); retval = perf_event_init_task(p); if (retval) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 78550c6187ed0..9dc2e236123b8 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -94,8 +94,8 @@ const char *task_event_names[] = {"PUT_PREV_TASK", "PICK_NEXT_TASK", "TASK_WAKE", "TASK_MIGRATE", "TASK_UPDATE", "IRQ_UPDATE"}; -ATOMIC_NOTIFIER_HEAD(migration_notifier_head); -ATOMIC_NOTIFIER_HEAD(load_alert_notifier_head); +//ATOMIC_NOTIFIER_HEAD(migration_notifier_head); +//ATOMIC_NOTIFIER_HEAD(load_alert_notifier_head); #ifdef smp_mb__before_atomic void __smp_mb__before_atomic(void) @@ -3360,9 +3360,9 @@ void set_numabalancing_state(bool enabled) /* * fork()/clone()-time setup: */ -void sched_fork(struct task_struct *p) +void sched_fork(unsigned long flags, struct task_struct *p) { - unsigned long flags; + //unsigned long flags; int cpu = get_cpu(); __sched_fork(p); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index e31e69332d5c8..1333338c98875 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -334,6 +334,7 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = sched_window_update_handler, }, +#ifdef TJK_HMP { .procname = "sched_heavy_task", .data = &sysctl_sched_heavy_task_pct, @@ -342,6 +343,7 @@ static struct ctl_table kern_table[] = { .proc_handler = sched_hmp_proc_update_handler, }, #endif +#endif #ifdef CONFIG_SCHED_HMP { .procname = "sched_account_wait_time", @@ -371,6 +373,7 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = sched_window_update_handler, }, +#ifdef TJK_HMP { .procname = "sched_small_task", .data = &sysctl_sched_small_task_pct, @@ -392,6 +395,7 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = sched_hmp_proc_update_handler, }, +#endif { .procname = "sched_spill_nr_run", .data = &sysctl_sched_spill_nr_run, @@ -400,6 +404,7 @@ static struct ctl_table kern_table[] = { .proc_handler = proc_dointvec_minmax, .extra1 = &zero, }, +#ifdef TJK_HMP { .procname = "sched_upmigrate", .data = &sysctl_sched_upmigrate_pct, @@ -435,6 +440,7 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = sched_hmp_proc_update_handler, }, +#endif { .procname = "sched_boost", .data = &sysctl_sched_boost, From d0962f4937ee44e8717aec6a666ae54df8d840f5 Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Tue, 15 Dec 2015 16:04:19 -0800 Subject: [PATCH 11/86] Backport EAS scheduler from 3.18 Change-Id: I0364a17809b0ecc94aa2cbd8cb8d1d10cea3c9af Signed-off-by: Joe Maples --- Makefile | 175 +- arch/arm64/include/asm/sysreg.h | 60 + include/linux/cgroup_subsys.h | 6 + include/linux/preempt.h | 2 +- include/linux/sched.h | 655 ++- include/linux/sched/deadline.h | 24 + include/linux/sched/prio.h | 60 + include/linux/sched/rt.h | 42 +- include/linux/sched/sysctl.h | 38 +- kernel/context_tracking.c | 2 + kernel/sched/Makefile | 8 +- kernel/sched/auto_group.c | 13 +- kernel/sched/clock.c | 112 +- kernel/sched/completion.c | 299 ++ kernel/sched/core.c | 7631 +++++++++++++----------------- kernel/sched/cpuacct.c | 20 +- kernel/sched/cpudeadline.c | 227 + kernel/sched/cpudeadline.h | 33 + kernel/sched/cpufreq_sched.c | 464 ++ kernel/sched/cpupri.c | 11 +- kernel/sched/cpupri.h | 2 +- kernel/sched/cputime.c | 89 +- kernel/sched/deadline.c | 1689 +++++++ kernel/sched/debug.c | 189 +- kernel/sched/fair.c | 7661 ++++++++++++++++--------------- kernel/sched/features.h | 34 +- kernel/sched/idle_task.c | 46 +- kernel/sched/proc.c | 584 +++ kernel/sched/rt.c | 549 +-- kernel/sched/sched.h | 908 ++-- kernel/sched/stats.c | 2 +- kernel/sched/stats.h | 90 +- kernel/sched/stop_task.c | 64 +- kernel/sched/tune.c | 720 +++ kernel/sched/tune.h | 35 + 35 files changed, 12874 insertions(+), 9670 deletions(-) create mode 100644 arch/arm64/include/asm/sysreg.h create mode 100644 include/linux/sched/deadline.h create mode 100644 include/linux/sched/prio.h create mode 100644 kernel/sched/completion.c create mode 100644 kernel/sched/cpudeadline.c create mode 100644 kernel/sched/cpudeadline.h create mode 100644 kernel/sched/cpufreq_sched.c create mode 100644 kernel/sched/deadline.c create mode 100644 kernel/sched/proc.c create mode 100644 kernel/sched/tune.c create mode 100644 kernel/sched/tune.h diff --git a/Makefile b/Makefile index f1055fe0e8d75..bf651e5ff2b8f 100755 --- a/Makefile +++ b/Makefile @@ -165,12 +165,7 @@ export srctree objtree VPATH # then ARCH is assigned, getting whatever value it gets normally, and # SUBARCH is subsequently ignored. -SUBARCH := $(shell uname -m | sed -e s/i.86/x86/ -e s/x86_64/x86/ \ - -e s/sun4u/sparc64/ \ - -e s/arm.*/arm/ -e s/sa110/arm/ \ - -e s/s390x/s390/ -e s/parisc64/parisc/ \ - -e s/ppc.*/powerpc/ -e s/mips.*/mips/ \ - -e s/sh[234].*/sh/ -e s/aarch64.*/arm64/ ) +SUBARCH := arm64 # Cross compiling and selecting different set of gcc/bin-utils # --------------------------------------------------------------------------- @@ -192,9 +187,13 @@ SUBARCH := $(shell uname -m | sed -e s/i.86/x86/ -e s/x86_64/x86/ \ # "make" in the configured kernel build directory always uses that. # Default value for CROSS_COMPILE is not to prefix executables # Note: Some architectures assign CROSS_COMPILE in their arch/*/Makefile -ARCH ?= $(SUBARCH) -CROSS_COMPILE ?= $(CONFIG_CROSS_COMPILE:"%"=%) - +ARCH ?= arm64 +# Check for Suzumiya Build System +ifeq ($(TOOLCHAIN_SUFFIX),) + CROSS_COMPILE ?= $(CONFIG_CROSS_COMPILE:"%"=%) +else + CROSS_COMPILE := $(TOOLCHAIN_SUFFIX) +endif # Architecture as present in compile.h UTS_MACHINE := $(ARCH) SRCARCH := $(ARCH) @@ -238,11 +237,39 @@ export KCONFIG_CONFIG CONFIG_SHELL := $(shell if [ -x "$$BASH" ]; then echo $$BASH; \ else if [ -x /bin/bash ]; then echo /bin/bash; \ else echo sh; fi ; fi) - -HOSTCC = gcc -HOSTCXX = g++ -HOSTCFLAGS = -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 -fomit-frame-pointer -std=gnu89 -HOSTCXXFLAGS = -O2 +FLAGS_OPTIMIZE := -falign-functions=32 -fgcse-las -fivopts \ + -fgcse-sm \ + -fipa-pta \ + -fomit-frame-pointer \ + -frename-registers \ + -ftracer \ + -ftree-loop-im \ + -ftree-loop-ivcanon \ + -funsafe-loop-optimizations \ + -funswitch-loops \ + -fweb \ + -Wno-error=array-bounds \ + -Wno-error=clobbered \ + -Wno-error=maybe-uninitialized \ + -Wno-error=strict-overflow \ + -fgcse-after-reload \ + -floop-block \ + -floop-interchange \ + -floop-nest-optimize \ + -floop-parallelize-all \ + -floop-strip-mine \ + -fmodulo-sched \ + -fmodulo-sched-allow-regmoves \ + -frerun-cse-after-loop \ + -frename-registers \ + $(GRAPHITE) + +GRAPHITE = -fgraphite -fgraphite-identity -floop-interchange -ftree-loop-distribution -floop-strip-mine -floop-block -ftree-loop-linear \ + $(FLAGS_OPTIMIZE) +HOSTCC = $(which ccache) gcc +HOSTCXX = $(which ccache) g++ +HOSTCFLAGS = -Wall -Wmissing-prototypes -Wstrict-prototypes -Ofast -fno-inline-functions -fno-ipa-cp-clone -fomit-frame-pointer -std=gnu89 $(GRAPHITE) $(FLAGS_OPTIMIZE) +HOSTCXXFLAGS = -Ofast -fno-inline-functions -fgcse-las -pipe -fno-ipa-cp-clone $(GRAPHITE) $(FLAGS_OPTIMIZE) # Decide whether to build built-in, modular, or both. # Normally, just do built-in. @@ -326,7 +353,13 @@ include $(srctree)/scripts/Kbuild.include AS = $(CROSS_COMPILE)as LD = $(CROSS_COMPILE)ld -REAL_CC = $(CROSS_COMPILE)gcc +CC = $(which ccache) $(CROSS_COMPILE)gcc +LD += -Ofast --strip-debug +CC += -Ofast +CC += -fmodulo-sched -fmodulo-sched-allow-regmoves +CC += -fgraphite-identity -floop-block -floop-interchange -floop-strip-mine +CC += -ftree-loop-linear -ftree-loop-distribution +CC += -mlow-precision-recip-sqrt -mpc-relative-literal-loads CPP = $(CC) -E AR = $(CROSS_COMPILE)ar NM = $(CROSS_COMPILE)nm @@ -340,20 +373,33 @@ DEPMOD = /sbin/depmod PERL = perl CHECK = sparse -# Use the wrapper for the compiler. This wrapper scans for new -# warnings and causes the build to stop upon encountering them. -CC = $(srctree)/scripts/gcc-wrapper.py $(REAL_CC) - CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \ -Wbitwise -Wno-return-void $(CF) -CFLAGS_MODULE = -AFLAGS_MODULE = -LDFLAGS_MODULE = -CFLAGS_KERNEL = -AFLAGS_KERNEL = +CFLAGS_MODULE = $(GRAPHITE) +AFLAGS_MODULE = $(GRAPHITE) +LDFLAGS_MODULE = --strip-debug +CFLAGS_KERNEL = $(GRAPHITE) -mcpu=cortex-a57.cortex-a53+crypto+crc -mtune=cortex-a57.cortex-a53 -march=armv8-a+crypto+crc +AFLAGS_KERNEL = $(GRAPHITE) CFLAGS_GCOV = -fprofile-arcs -ftest-coverage +# fall back to -march=armv8-a in case the compiler isn't compatible +# with -mcpu and -mtune +ARM_ARCH_OPT := -mcpu=cortex-a57.cortex-a53+crypto+crc -mtune=cortex-a57.cortex-a53 +GEN_OPT_FLAGS := $(call cc-option,$(ARM_ARCH_OPT),-march=armv8-a+crypto+crc) \ + -g0 \ + -DNDEBUG \ + -fomit-frame-pointer \ + -fmodulo-sched \ + -fmodulo-sched-allow-regmoves \ + -fivopts \ + -fsection-anchors \ + -Wno-array-bounds \ + -fno-store-merging \ + -pipe \ + $(GRAPHITE) \ + $(FLAGS_OPTIMIZE) + # Use USERINCLUDE when you must reference the UAPI directories only. USERINCLUDE := \ -I$(srctree)/arch/$(hdr-arch)/include/uapi \ @@ -377,16 +423,37 @@ KBUILD_CFLAGS := -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \ -fno-strict-aliasing -fno-common \ -Werror-implicit-function-declaration \ -Wno-format-security \ + -mcpu=cortex-a57.cortex-a53+crypto+crc -mtune=cortex-a57.cortex-a53+crypto+crc \ + -march=armv8-a+crypto+crc \ + -no-pie -fno-pic \ + -Ofast -fno-inline-functions \ + -fgcse-sm -fsched-spec-load \ + -fsingle-precision-constant \ -fno-delete-null-pointer-checks \ - -std=gnu89 - -KBUILD_AFLAGS_KERNEL := -KBUILD_CFLAGS_KERNEL := + -std=gnu89 -Wno-unused-const-variable -Wno-misleading-indentation \ + -Wno-memset-transposed-args -Wno-bool-compare -Wno-logical-not-parentheses \ + -Wno-switch-bool \ + -Wno-bool-operation -Wno-nonnull -Wno-switch-unreachable -Wno-format-truncation -Wno-format-overflow -Wno-duplicate-decl-specifier -Wno-memset-elt-size -Wno-int-in-bool-context \ + $(GEN_OPT_FLAGS) \ + $(GRAPHITE) \ + $(FLAGS_OPTIMIZE) + +KBUILD_AFLAGS_KERNEL := $(GEN_OPT_FLAGS) +KBUILD_CFLAGS_KERNEL := $(GEN_OPT_FLAGS) KBUILD_AFLAGS := -D__ASSEMBLY__ -KBUILD_AFLAGS_MODULE := -DMODULE -KBUILD_CFLAGS_MODULE := -DMODULE +KBUILD_AFLAGS_MODULE := -DMODULE $(GEN_OPT_FLAGS) +KBUILD_CFLAGS_MODULE := -DMODULE -fno-pic $(GEN_OPT_FLAGS) KBUILD_LDFLAGS_MODULE := -T $(srctree)/scripts/module-common.lds +# Disable some warnings +KBUILD_CFLAGS += $(call cc-disable-warning,maybe-uninitialized) +KBUILD_CFLAGS += $(call cc-disable-warning,array-bounds) +KBUILD_CFLAGS += $(call cc-disable-warning,unused-function) +KBUILD_CFLAGS += $(call cc-disable-warning, unused-variable) + +# another warning +KBUILD_CFLAGS += $(call cc-disable-warning, unused-const-variable) + #ifdef VENDOR_EDIT KBUILD_CFLAGS += -DVENDOR_EDIT KBUILD_CPPFLAGS += -DVENDOR_EDIT @@ -398,6 +465,9 @@ CFLAGS_MODULE += -DVENDOR_EDIT KERNELRELEASE = $(shell cat include/config/kernel.release 2> /dev/null) KERNELVERSION = $(VERSION)$(if $(PATCHLEVEL),.$(PATCHLEVEL)$(if $(SUBLEVEL),.$(SUBLEVEL)))$(EXTRAVERSION) +# arter97's optimizations +KBUILD_CFLAGS += -pipe -fno-pic + export VERSION PATCHLEVEL SUBLEVEL KERNELRELEASE KERNELVERSION export ARCH SRCARCH CONFIG_SHELL HOSTCC HOSTCFLAGS CROSS_COMPILE AS LD CC export CPP AR NM STRIP OBJCOPY OBJDUMP @@ -504,7 +574,7 @@ ifeq ($(config-targets),1) # Read arch specific Makefile to set KBUILD_DEFCONFIG as needed. # KBUILD_DEFCONFIG may point out an alternative default configuration -# used for 'make defconfig' +# used for 'make defconfig'. include $(srctree)/arch/$(SRCARCH)/Makefile export KBUILD_DEFCONFIG KBUILD_KCONFIG @@ -583,11 +653,15 @@ endif # $(dot-config) # Defaults to vmlinux, but the arch makefile usually adds further targets all: vmlinux -ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE -KBUILD_CFLAGS += -Os $(call cc-disable-warning,maybe-uninitialized,) -else -KBUILD_CFLAGS += -O2 -endif +# force no-pie for distro compilers that enable pie by default +KBUILD_CFLAGS += $(call cc-option, -fno-pie) +KBUILD_CFLAGS += $(call cc-option, -no-pie) +KBUILD_AFLAGS += $(call cc-option, -fno-pie) +KBUILD_CPPFLAGS += $(call cc-option, -fno-pie) + + +KBUILD_CFLAGS += -Ofast -fno-inline-functions -fno-pic -fno-ipa-cp-clone -Wno-maybe-uninitialized +KBUILD_CFLAGS += $(call cc-disable-warning,maybe-uninitialized,) include $(srctree)/arch/$(SRCARCH)/Makefile @@ -601,10 +675,6 @@ KBUILD_CFLAGS += $(call cc-option,-fno-reorder-blocks,) \ $(call cc-option,-fno-partial-inlining) endif -ifneq ($(CONFIG_FRAME_WARN),0) -KBUILD_CFLAGS += $(call cc-option,-Wframe-larger-than=${CONFIG_FRAME_WARN}) -endif - # Handle stack protector mode. ifdef CONFIG_CC_STACKPROTECTOR_REGULAR stackp-flag := -fstack-protector @@ -629,19 +699,7 @@ KBUILD_CFLAGS += $(stackp-flag) # This warning generated too much noise in a regular build. # Use make W=1 to enable this warning (see scripts/Makefile.build) KBUILD_CFLAGS += $(call cc-disable-warning, unused-but-set-variable) - -ifdef CONFIG_FRAME_POINTER -KBUILD_CFLAGS += -fno-omit-frame-pointer -fno-optimize-sibling-calls -else -# Some targets (ARM with Thumb2, for example), can't be built with frame -# pointers. For those, we don't have FUNCTION_TRACER automatically -# select FRAME_POINTER. However, FUNCTION_TRACER adds -pg, and this is -# incompatible with -fomit-frame-pointer with current GCC, so we don't use -# -fomit-frame-pointer with FUNCTION_TRACER. -ifndef CONFIG_FUNCTION_TRACER KBUILD_CFLAGS += -fomit-frame-pointer -endif -endif KBUILD_CFLAGS += $(call cc-option, -fno-var-tracking-assignments) @@ -655,19 +713,6 @@ KBUILD_CFLAGS += $(call cc-option, -femit-struct-debug-baseonly) \ $(call cc-option,-fno-var-tracking) endif -ifdef CONFIG_FUNCTION_TRACER -ifdef CONFIG_HAVE_FENTRY -CC_USING_FENTRY := $(call cc-option, -mfentry -DCC_USING_FENTRY) -endif -KBUILD_CFLAGS += -pg $(CC_USING_FENTRY) -KBUILD_AFLAGS += $(CC_USING_FENTRY) -ifdef CONFIG_DYNAMIC_FTRACE - ifdef CONFIG_HAVE_C_RECORDMCOUNT - BUILD_C_RECORDMCOUNT := y - export BUILD_C_RECORDMCOUNT - endif -endif -endif # We trigger additional mismatches with less inlining ifdef CONFIG_DEBUG_SECTION_MISMATCH diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h new file mode 100644 index 0000000000000..5c89df0acbcb2 --- /dev/null +++ b/arch/arm64/include/asm/sysreg.h @@ -0,0 +1,60 @@ +/* + * Macros for accessing system registers with older binutils. + * + * Copyright (C) 2014 ARM Ltd. + * Author: Catalin Marinas + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __ASM_SYSREG_H +#define __ASM_SYSREG_H + +#define sys_reg(op0, op1, crn, crm, op2) \ + ((((op0)-2)<<19)|((op1)<<16)|((crn)<<12)|((crm)<<8)|((op2)<<5)) + +#ifdef __ASSEMBLY__ + + .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 + .equ __reg_num_x\num, \num + .endr + .equ __reg_num_xzr, 31 + + .macro mrs_s, rt, sreg + .inst 0xd5300000|(\sreg)|(__reg_num_\rt) + .endm + + .macro msr_s, sreg, rt + .inst 0xd5100000|(\sreg)|(__reg_num_\rt) + .endm + +#else + +asm( +" .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30\n" +" .equ __reg_num_x\\num, \\num\n" +" .endr\n" +" .equ __reg_num_xzr, 31\n" +"\n" +" .macro mrs_s, rt, sreg\n" +" .inst 0xd5300000|(\\sreg)|(__reg_num_\\rt)\n" +" .endm\n" +"\n" +" .macro msr_s, sreg, rt\n" +" .inst 0xd5100000|(\\sreg)|(__reg_num_\\rt)\n" +" .endm\n" +); + +#endif + +#endif /* __ASM_SYSREG_H */ diff --git a/include/linux/cgroup_subsys.h b/include/linux/cgroup_subsys.h index e5e6b0dd41d9e..61fefd93fafe4 100644 --- a/include/linux/cgroup_subsys.h +++ b/include/linux/cgroup_subsys.h @@ -31,6 +31,12 @@ SUBSYS(cpuacct) /* */ +#if IS_SUBSYS_ENABLED(CONFIG_CGROUP_SCHEDTUNE) +SUBSYS(schedtune) +#endif + +/* */ + #if IS_SUBSYS_ENABLED(CONFIG_MEMCG) SUBSYS(mem_cgroup) #endif diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 9a66079428676..674ff1d7fd7f8 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -10,7 +10,7 @@ #include #include -#if 1 && (defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER)) +#if 0 && (defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER)) extern void add_preempt_count(int val); extern void sub_preempt_count(int val); #else diff --git a/include/linux/sched.h b/include/linux/sched.h index 70ad9d64cb031..fe5132d5a04af 100755 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -3,6 +3,8 @@ #include +#include + struct sched_param { int sched_priority; @@ -16,19 +18,24 @@ struct sched_param { #include #include #include +#include #include #include #include #include #include #include +#include #include #include -#include +#include #include #include +#ifdef TJK_HMP +#include +#endif #include #include #include @@ -52,6 +59,7 @@ struct sched_param { #include #include #include +#include #include @@ -96,6 +104,10 @@ struct sched_param { * Given this task model, there are a multiplicity of scheduling algorithms * and policies, that can be used to ensure all the tasks will make their * timing constraints. + * + * As of now, the SCHED_DEADLINE policy (sched_dl scheduling class) is the + * only user of this new interface. More information about the algorithm + * available in the scheduling class file or in Documentation/. */ struct sched_attr { u32 size; @@ -122,12 +134,11 @@ struct bio_list; struct fs_struct; struct perf_event_context; struct blk_plug; +struct filename; -/* - * List of flags we want to share for kernel threads, - * if only because they are not used by them anyway. - */ -#define CLONE_KERNEL (CLONE_FS | CLONE_FILES | CLONE_SIGHAND) +#define VMACACHE_BITS 2 +#define VMACACHE_SIZE (1U << VMACACHE_BITS) +#define VMACACHE_MASK (VMACACHE_SIZE - 1) /* * These are the constant used to fake the fixed-point load-average @@ -159,9 +170,10 @@ extern int nr_threads; DECLARE_PER_CPU(unsigned long, process_counts); extern int nr_processes(void); extern unsigned long nr_running(void); +extern bool single_task_running(void); extern unsigned long nr_iowait(void); extern unsigned long nr_iowait_cpu(int cpu); -extern unsigned long this_cpu_load(void); +extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load); extern void sched_update_nr_prod(int cpu, long delta, bool inc); extern void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg); @@ -169,14 +181,6 @@ extern void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg); extern void calc_global_load(unsigned long ticks); extern void update_cpu_load_nohz(void); -/* Notifier for when a task gets migrated to a new CPU */ -struct task_migration_notifier { - struct task_struct *task; - int from_cpu; - int to_cpu; -}; -extern void register_task_migration_notifier(struct notifier_block *n); - extern unsigned long get_parent_ip(unsigned long addr); extern void dump_cpu_task(int cpu); @@ -207,8 +211,9 @@ print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq); #define __TASK_STOPPED 4 #define __TASK_TRACED 8 /* in tsk->exit_state */ -#define EXIT_ZOMBIE 16 -#define EXIT_DEAD 32 +#define EXIT_DEAD 16 +#define EXIT_ZOMBIE 32 +#define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD) /* in tsk->state again */ #define TASK_DEAD 64 #define TASK_WAKEKILL 128 @@ -216,7 +221,7 @@ print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq); #define TASK_PARKED 512 #define TASK_STATE_MAX 1024 -#define TASK_STATE_TO_CHAR_STR "RSDTtZXxKWP" +#define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWP" extern char ___assert_task_state[1 - 2*!!( sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1)]; @@ -233,11 +238,10 @@ extern char ___assert_task_state[1 - 2*!!( /* get_task_state() */ #define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \ TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \ - __TASK_TRACED) + __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD) #define task_is_traced(task) ((task->state & __TASK_TRACED) != 0) #define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0) -#define task_is_dead(task) ((task)->exit_state != 0) #define task_is_stopped_or_traced(task) \ ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0) #define task_contributes_to_load(task) \ @@ -311,6 +315,10 @@ extern int get_nohz_timer_target(void); #else static inline void nohz_balance_enter_idle(int cpu) { } static inline void set_cpu_sd_state_idle(void) { } +static inline int get_nohz_timer_target(void) +{ + return smp_processor_id(); +} #endif /* @@ -366,6 +374,14 @@ static inline void lockup_detector_init(void) } #endif +#ifdef CONFIG_DETECT_HUNG_TASK +void reset_hung_task_detector(void); +#else +static inline void reset_hung_task_detector(void) +{ +} +#endif + /* Attach to any functions which should be ignored in wchan output. */ #define __sched __attribute__((__section__(".sched.text"))) @@ -399,22 +415,38 @@ arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, static inline void arch_pick_mmap_layout(struct mm_struct *mm) {} #endif - -extern void set_dumpable(struct mm_struct *mm, int value); -extern int get_dumpable(struct mm_struct *mm); - #define SUID_DUMP_DISABLE 0 /* No setuid dumping */ #define SUID_DUMP_USER 1 /* Dump as user of process */ #define SUID_DUMP_ROOT 2 /* Dump as root */ /* mm flags */ -/* dumpable bits */ -#define MMF_DUMPABLE 0 /* core dump is permitted */ -#define MMF_DUMP_SECURELY 1 /* core file is readable only by root */ +/* for SUID_DUMP_* above */ +#define MMF_DUMPABLE 0 +#define MMF_DUMP_SECURELY 1 #define MMF_DUMPABLE_BITS 2 #define MMF_DUMPABLE_MASK ((1 << MMF_DUMPABLE_BITS) - 1) +extern void set_dumpable(struct mm_struct *mm, int value); +extern int get_dumpable(struct mm_struct *mm); +/* + * This returns the actual value of the suid_dumpable flag. For things + * that are using this for checking for privilege transitions, it must + * test against SUID_DUMP_USER rather than treating it as a boolean + * value. + */ +#if 0 +static inline int __get_dumpable(unsigned long mm_flags) +{ + return mm_flags & MMF_DUMPABLE_MASK; +} + +static inline int get_dumpable(struct mm_struct *mm) +{ + return __get_dumpable(mm->flags); +} +#endif + /* coredump filter bits */ #define MMF_DUMP_ANON_PRIVATE 2 #define MMF_DUMP_ANON_SHARED 3 @@ -512,6 +544,14 @@ struct task_cputime { .sum_exec_runtime = 0, \ } +#ifdef TJK_HMP +#ifdef CONFIG_PREEMPT_COUNT +#define PREEMPT_DISABLED (1 + PREEMPT_ENABLED) +#else +#define PREEMPT_DISABLED PREEMPT_ENABLED +#endif +#endif + /* * Disable preemption until the scheduler is running. * Reset by start_kernel()->sched_init()->init_idle(). @@ -630,6 +670,7 @@ struct signal_struct { * Live threads maintain their own counters and add to these * in __exit_signal, except for the group leader. */ + seqlock_t stats_lock; cputime_t utime, stime, cutime, cstime; cputime_t gtime; cputime_t cgtime; @@ -724,7 +765,7 @@ static inline int signal_group_exit(const struct signal_struct *sig) struct user_struct { atomic_t __count; /* reference count */ atomic_t processes; /* How many processes does this user have? */ - atomic_t files; /* How many open files does this user have? */ + atomic_t files; atomic_t sigpending; /* How many pending signals does this user have? */ #ifdef CONFIG_INOTIFY_USER atomic_t inotify_watches; /* How many inotify watches does this user have? */ @@ -801,7 +842,7 @@ struct task_delay_info { * associated with the operation is added to XXX_delay. * XXX_delay contains the accumulated delay time in nanoseconds. */ - struct timespec blkio_start, blkio_end; /* Shared by blkio, swapin */ + u64 blkio_start; /* Shared by blkio, swapin */ u64 blkio_delay; /* wait for sync block io completion */ u64 swapin_delay; /* wait for swapin block io completion */ u32 blkio_count; /* total count of the number of sync block */ @@ -809,7 +850,7 @@ struct task_delay_info { u32 swapin_count; /* total count of the number of swapin block */ /* io operations performed */ - struct timespec freepages_start, freepages_end; + u64 freepages_start; u64 freepages_delay; /* wait for memory reclaim */ u32 freepages_count; /* total count of memory reclaim */ }; @@ -835,7 +876,7 @@ enum cpu_idle_type { }; /* - * Increase resolution of cpu_power calculations + * Increase resolution of cpu_capacity calculations */ #define SCHED_CAPACITY_SHIFT 10 #define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT) @@ -850,14 +891,36 @@ enum cpu_idle_type { #define SD_BALANCE_FORK 0x0008 /* Balance on fork, clone */ #define SD_BALANCE_WAKE 0x0010 /* Balance on wakeup */ #define SD_WAKE_AFFINE 0x0020 /* Wake task to waking CPU */ -#define SD_SHARE_CPUPOWER 0x0080 /* Domain members share cpu power */ +#define SD_SHARE_CPUCAPACITY 0x0080 /* Domain members share cpu power */ +#define SD_SHARE_POWERDOMAIN 0x0100 /* Domain members share power domain */ #define SD_SHARE_PKG_RESOURCES 0x0200 /* Domain members share cpu pkg resources */ #define SD_SERIALIZE 0x0400 /* Only a single load balancing instance */ #define SD_ASYM_PACKING 0x0800 /* Place busy groups earlier in the domain */ #define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */ #define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */ +#define SD_NUMA 0x4000 /* cross-node balancing */ +#define SD_SHARE_CAP_STATES 0x8000 /* Domain members share capacity state */ + +#ifdef CONFIG_SCHED_SMT +static inline int cpu_smt_flags(void) +{ + return SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES; +} +#endif -extern int __weak arch_sd_sibiling_asym_packing(void); +#ifdef CONFIG_SCHED_MC +static inline int cpu_core_flags(void) +{ + return SD_SHARE_PKG_RESOURCES; +} +#endif + +#ifdef CONFIG_NUMA +static inline int cpu_numa_flags(void) +{ + return SD_NUMA; +} +#endif struct sched_domain_attr { int relax_domain_level; @@ -887,6 +950,8 @@ struct sched_group_energy { struct capacity_state *cap_states; /* ptr to capacity state array */ }; +unsigned long capacity_curr_of(int cpu); + struct sched_group; struct sched_domain { @@ -915,7 +980,9 @@ struct sched_domain { unsigned int balance_interval; /* initialise to 1. units in ms. */ unsigned int nr_balance_failed; /* initialise to 0 */ - u64 last_update; + /* idle_balance() stats */ + u64 max_newidle_lb_cost; + unsigned long next_decay_max_lb_cost; #ifdef CONFIG_SCHEDSTATS /* load_balance() stats */ @@ -981,6 +1048,42 @@ void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms); bool cpus_share_cache(int this_cpu, int that_cpu); +typedef const struct cpumask *(*sched_domain_mask_f)(int cpu); +typedef int (*sched_domain_flags_f)(void); +typedef const struct sched_group_energy *(*sched_domain_energy_f)(int cpu); + +#define SDTL_OVERLAP 0x01 + +struct sd_data { + struct sched_domain **__percpu sd; + struct sched_group **__percpu sg; + struct sched_group_capacity **__percpu sgc; + struct sched_group_energy **__percpu sge; +}; + +struct sched_domain_topology_level { + sched_domain_mask_f mask; + sched_domain_flags_f sd_flags; + sched_domain_energy_f energy; + int flags; + int numa_level; + struct sd_data data; +#ifdef CONFIG_SCHED_DEBUG + char *name; +#endif +}; + +extern struct sched_domain_topology_level *sched_domain_topology; + +extern void set_sched_topology(struct sched_domain_topology_level *tl); +extern void wake_up_if_idle(int cpu); + +#ifdef CONFIG_SCHED_DEBUG +# define SD_INIT_NAME(type) .name = #type +#else +# define SD_INIT_NAME(type) +#endif + #else /* CONFIG_SMP */ struct sched_domain_attr; @@ -1014,24 +1117,35 @@ struct pipe_inode_info; struct uts_namespace; struct load_weight { - unsigned long weight, inv_weight; + unsigned long weight; + u32 inv_weight; }; struct sched_avg { - /* - * These sums represent an infinite geometric series and so are bound - * above by 1024/(1-y). Thus we only need a u32 to store them for for all - * choices of y < 1-2^(-32)*1024. - */ - u32 runnable_avg_sum, runnable_avg_period; -#ifdef CONFIG_SCHED_HMP - u32 runnable_avg_sum_scaled; -#endif u64 last_runnable_update; s64 decay_count; - unsigned long load_avg_contrib; + /* + * utilization_avg_contrib describes the amount of time that a + * sched_entity is running on a CPU. It is based on running_avg_sum + * and is scaled in the range [0..SCHED_LOAD_SCALE]. + * load_avg_contrib described the amount of time that a sched_entity + * is runnable on a rq. It is based on both runnable_avg_sum and the + * weight of the task. + */ + unsigned long load_avg_contrib, utilization_avg_contrib; + /* + * These sums represent an infinite geometric series and so are bound + * above by 1024/(1-y). Thus we only need a u32 to store them for all + * choices of y < 1-2^(-32)*1024. + * running_avg_sum reflects the time that the sched_entity is + * effectively running on the CPU. + * runnable_avg_sum represents the amount of time a sched_entity is on + * a runqueue which includes the running time that is monitored by + * running_avg_sum. + */ + u32 runnable_avg_sum, avg_period, running_avg_sum; }; - + #ifdef CONFIG_SCHEDSTATS struct sched_statistics { u64 wait_start; @@ -1102,6 +1216,7 @@ struct ravg { #endif #ifdef CONFIG_SCHED_FREQ_INPUT u32 curr_window, prev_window; + u16 active_windows; #endif }; @@ -1123,6 +1238,7 @@ struct sched_entity { #endif #ifdef CONFIG_FAIR_GROUP_SCHED + int depth; struct sched_entity *parent; /* rq on which this entity is (to be) queued: */ struct cfs_rq *cfs_rq; @@ -1130,12 +1246,7 @@ struct sched_entity { struct cfs_rq *my_q; #endif -/* - * Load-tracking only depends on SMP, FAIR_GROUP_SCHED dependency below may be - * removed when useful for applications beyond shares distribution (e.g. - * load-balance). - */ -#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED) +#ifdef CONFIG_SMP /* Per-entity load-tracking */ struct sched_avg avg; #endif @@ -1157,7 +1268,62 @@ struct sched_rt_entity { #endif }; +struct sched_dl_entity { + struct rb_node rb_node; + /* + * Original scheduling parameters. Copied here from sched_attr + * during sched_setattr(), they will remain the same until + * the next sched_setattr(). + */ + u64 dl_runtime; /* maximum runtime for each instance */ + u64 dl_deadline; /* relative deadline of each instance */ + u64 dl_period; /* separation of two instances (period) */ + u64 dl_bw; /* dl_runtime / dl_deadline */ + + /* + * Actual scheduling parameters. Initialized with the values above, + * they are continously updated during task execution. Note that + * the remaining runtime could be < 0 in case we are in overrun. + */ + s64 runtime; /* remaining runtime for this instance */ + u64 deadline; /* absolute deadline for this instance */ + unsigned int flags; /* specifying the scheduler behaviour */ + + /* + * Some bool flags: + * + * @dl_throttled tells if we exhausted the runtime. If so, the + * task has to wait for a replenishment to be performed at the + * next firing of dl_timer. + * + * @dl_new tells if a new instance arrived. If so we must + * start executing it with full runtime and reset its absolute + * deadline; + * + * @dl_boosted tells if we are boosted due to DI. If so we are + * outside bandwidth enforcement mechanism (but only until we + * exit the critical section); + * + * @dl_yielded tells if task gave up the cpu before consuming + * all its available runtime during the last job. + */ + int dl_throttled, dl_new, dl_boosted, dl_yielded; + + /* + * Bandwidth enforcement timer. Each -deadline task has its + * own bandwidth to be enforced, thus we need one timer per task. + */ + struct hrtimer dl_timer; +}; + +union rcu_special { + struct { + bool blocked; + bool need_qs; + } b; + short s; +}; struct rcu_node; enum perf_event_task_context { @@ -1181,6 +1347,11 @@ struct task_struct { #ifdef CONFIG_SMP struct llist_node wake_entry; int on_cpu; + struct task_struct *last_wakee; + unsigned long wakee_flips; + unsigned long wakee_flip_decay_ts; + + int wake_cpu; #endif int on_rq; @@ -1196,26 +1367,21 @@ struct task_struct { * of this task */ u32 init_load_pct; + u64 last_wake_ts; +#ifdef CONFIG_SCHED_QHMP u64 run_start; #endif +#endif #ifdef CONFIG_CGROUP_SCHED struct task_group *sched_task_group; #endif + struct sched_dl_entity dl; #ifdef CONFIG_PREEMPT_NOTIFIERS /* list of struct preempt_notifier: */ struct hlist_head preempt_notifiers; #endif - /* - * fpu_counter contains the number of consecutive context switches - * that the FPU is used. If this is over a threshold, the lazy fpu - * saving becomes unlazy to save the trap. This is an unsigned char - * so that after 256 times the counter wraps and the behavior turns - * lazy again; this to deal with bursty apps that only use FPU for - * a short time - */ - unsigned char fpu_counter; #ifdef CONFIG_BLK_DEV_IO_TRACE unsigned int btrace_seq; #endif @@ -1232,9 +1398,12 @@ struct task_struct { #ifdef CONFIG_TREE_PREEMPT_RCU struct rcu_node *rcu_blocked_node; #endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */ -#ifdef CONFIG_RCU_BOOST - struct rt_mutex *rcu_boost_mutex; -#endif /* #ifdef CONFIG_RCU_BOOST */ +#ifdef CONFIG_TASKS_RCU + unsigned long rcu_tasks_nvcsw; + bool rcu_tasks_holdout; + struct list_head rcu_tasks_holdout_list; + int rcu_tasks_idle_cpu; +#endif /* #ifdef CONFIG_TASKS_RCU */ #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) struct sched_info sched_info; @@ -1243,12 +1412,16 @@ struct task_struct { struct list_head tasks; #ifdef CONFIG_SMP struct plist_node pushable_tasks; + struct rb_node pushable_dl_tasks; #endif struct mm_struct *mm, *active_mm; #ifdef CONFIG_COMPAT_BRK unsigned brk_randomized:1; #endif + /* per-thread vma caching */ + u32 vmacache_seqnum; + struct vm_area_struct *vmacache[VMACACHE_SIZE]; #if defined(SPLIT_RSS_COUNTING) struct task_rss_stat rss_stat; #endif @@ -1326,8 +1499,8 @@ struct task_struct { } vtime_snap_whence; #endif unsigned long nvcsw, nivcsw; /* context switch counts */ - struct timespec start_time; /* monotonic time */ - struct timespec real_start_time; /* boot based time */ + struct timespec start_time; /* monotonic time in nsec */ + struct timespec real_start_time; /* boot based time in nsec */ /* mm fault and swap info: this can arguably be seen as either mm-specific or thread-specific */ unsigned long min_flt, maj_flt; @@ -1348,6 +1521,7 @@ struct task_struct { #ifdef CONFIG_SYSVIPC /* ipc stuff */ struct sysv_sem sysvsem; + struct sysv_shm sysvshm; #endif #ifdef CONFIG_DETECT_HUNG_TASK /* hung task detection */ @@ -1479,6 +1653,9 @@ struct task_struct { struct mutex perf_event_mutex; struct list_head perf_event_list; #endif +#ifdef CONFIG_DEBUG_PREEMPT + unsigned long preempt_disable_ip; +#endif #ifdef CONFIG_NUMA struct mempolicy *mempolicy; /* Protected by alloc_lock */ short il_next; @@ -1486,10 +1663,49 @@ struct task_struct { #endif #ifdef CONFIG_NUMA_BALANCING int numa_scan_seq; - int numa_migrate_seq; unsigned int numa_scan_period; + unsigned int numa_scan_period_max; + int numa_preferred_nid; + unsigned long numa_migrate_retry; u64 node_stamp; /* migration stamp */ + u64 last_task_numa_placement; + u64 last_sum_exec_runtime; struct callback_head numa_work; + + struct list_head numa_entry; + struct numa_group *numa_group; + + /* + * Exponential decaying average of faults on a per-node basis. + * Scheduling placement decisions are made based on the these counts. + * The values remain static for the duration of a PTE scan + */ + unsigned long *numa_faults_memory; + unsigned long total_numa_faults; + + /* + * numa_faults_buffer records faults per node during the current + * scan window. When the scan completes, the counts in + * numa_faults_memory decay and these values are copied. + */ + unsigned long *numa_faults_buffer_memory; + + /* + * Track the nodes the process was running on when a NUMA hinting + * fault was incurred. + */ + unsigned long *numa_faults_cpu; + unsigned long *numa_faults_buffer_cpu; + + /* + * numa_faults_locality tracks if faults recorded during the last + * scan window were remote/local. The task scan period is adapted + * based on the locality of the faults with different weights + * depending on whether they were shared or private faults + */ + unsigned long numa_faults_locality[2]; + + unsigned long numa_pages_migrated; #endif /* CONFIG_NUMA_BALANCING */ struct rcu_head rcu; @@ -1526,6 +1742,9 @@ struct task_struct { unsigned long timer_slack_ns; unsigned long default_timer_slack_ns; +#ifdef CONFIG_KASAN + unsigned int kasan_depth; +#endif #ifdef CONFIG_FUNCTION_GRAPH_TRACER /* Index of current stored address in ret_stack */ int curr_ret_stack; @@ -1548,12 +1767,6 @@ struct task_struct { unsigned long trace_recursion; #endif /* CONFIG_TRACING */ #ifdef CONFIG_MEMCG /* memcg uses this to do batch job */ - struct memcg_batch_info { - int do_batch; /* incremented when batch uncharge started */ - struct mem_cgroup *memcg; /* target memcg of uncharge */ - unsigned long nr_pages; /* uncharged usage */ - unsigned long memsw_nr_pages; /* uncharged mem+swap usage */ - } memcg_batch; unsigned int memcg_kmem_skip_account; struct memcg_oom_info { struct mem_cgroup *memcg; @@ -1562,9 +1775,6 @@ struct task_struct { unsigned int may_oom:1; } memcg_oom; #endif -#ifdef CONFIG_HAVE_HW_BREAKPOINT - atomic_t ptrace_bp_refcnt; -#endif #ifdef CONFIG_UPROBES struct uprobe_task *utask; #endif @@ -1577,16 +1787,37 @@ struct task_struct { /* Future-safe accessor for struct task_struct's cpus_allowed. */ #define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) +#define TNF_MIGRATED 0x01 +#define TNF_NO_GROUP 0x02 +#define TNF_SHARED 0x04 +#define TNF_FAULT_LOCAL 0x08 + #ifdef CONFIG_NUMA_BALANCING -extern void task_numa_fault(int node, int pages, bool migrated); +extern void task_numa_fault(int node, int pages, int migrated); +extern pid_t task_numa_group_id(struct task_struct *p); extern void set_numabalancing_state(bool enabled); +extern void task_numa_free(struct task_struct *p); +extern bool should_numa_migrate_memory(struct task_struct *p, struct page *page, + int src_nid, int dst_cpu); #else -static inline void task_numa_fault(int node, int pages, bool migrated) +static inline void task_numa_fault(int node, int pages, int migrated) { } +static inline pid_t task_numa_group_id(struct task_struct *p) +{ + return 0; +} static inline void set_numabalancing_state(bool enabled) { } +static inline void task_numa_free(struct task_struct *p) +{ +} +static inline bool should_numa_migrate_memory(struct task_struct *p, + struct page *page, int src_nid, int dst_cpu) +{ + return true; +} #endif static inline struct pid *task_pid(struct task_struct *task) @@ -1662,6 +1893,24 @@ static inline pid_t task_tgid_vnr(struct task_struct *tsk) } +static inline int pid_alive(const struct task_struct *p); +static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns) +{ + pid_t pid = 0; + + rcu_read_lock(); + if (pid_alive(tsk)) + pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns); + rcu_read_unlock(); + + return pid; +} + +static inline pid_t task_ppid_nr(const struct task_struct *tsk) +{ + return task_ppid_nr_ns(tsk, &init_pid_ns); +} + static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns) { @@ -1698,8 +1947,10 @@ static inline pid_t task_pgrp_nr(struct task_struct *tsk) * Test if a process is not yet dead (at most zombie state) * If pid_alive fails, then pointers within the task structure * can be stale and must not be dereferenced. + * + * Return: 1 if the process is alive. 0 otherwise. */ -static inline int pid_alive(struct task_struct *p) +static inline int pid_alive(const struct task_struct *p) { return p->pids[PIDTYPE_PID].pid != NULL; } @@ -1709,6 +1960,8 @@ static inline int pid_alive(struct task_struct *p) * @tsk: Task structure to be checked. * * Check if a task structure is the first user space task the kernel created. + * + * Return: 1 if the task structure is init. 0 otherwise. */ static inline int is_global_init(struct task_struct *tsk) { @@ -1764,10 +2017,26 @@ extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, extern int task_free_register(struct notifier_block *n); extern int task_free_unregister(struct notifier_block *n); -#ifdef CONFIG_SCHED_FREQ_INPUT + +struct sched_load { + unsigned long prev_load; + unsigned long new_task_load; +}; + +#if defined(CONFIG_SCHED_FREQ_INPUT) extern int sched_set_window(u64 window_start, unsigned int window_size); extern unsigned long sched_get_busy(int cpu); +extern void sched_get_cpus_busy(struct sched_load *busy, + const struct cpumask *query_cpus); extern void sched_set_io_is_busy(int val); +#ifdef CONFIG_SCHED_QHMP +static inline int sched_update_freq_max_load(const cpumask_t *cpumask) +{ + return 0; +} +#else +int sched_update_freq_max_load(const cpumask_t *cpumask); +#endif #else static inline int sched_set_window(u64 window_start, unsigned int window_size) { @@ -1777,12 +2046,20 @@ static inline unsigned long sched_get_busy(int cpu) { return 0; } +static inline void sched_get_cpus_busy(struct sched_load *busy, + const struct cpumask *query_cpus) {}; static inline void sched_set_io_is_busy(int val) {}; + +static inline int sched_update_freq_max_load(const cpumask_t *cpumask) +{ + return 0; +} #endif /* * Per process flags */ +#define PF_WAKE_UP_IDLE 0x00000002 /* try to wake up on an idle CPU */ #define PF_EXITING 0x00000004 /* getting shut down */ #define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */ #define PF_VCPU 0x00000010 /* I'm a virtual CPU */ @@ -1805,14 +2082,13 @@ static inline void sched_set_io_is_busy(int val) {}; #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ #define PF_RANDOMIZE 0x00400000 /* randomize virtual address space */ #define PF_SWAPWRITE 0x00800000 /* Allowed to write to swap */ -#define PF_SPREAD_PAGE 0x01000000 /* Spread page cache over cpuset */ -#define PF_SPREAD_SLAB 0x02000000 /* Spread some slab caches over cpuset */ #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_allowed */ #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */ -#define PF_MEMPOLICY 0x10000000 /* Non-default NUMA mempolicy */ +#define PF_SPREAD_PAGE 0x01000000 +#define PF_SPREAD_SLAB 0x02000000 #define PF_MUTEX_TESTER 0x20000000 /* Thread belongs to the rt mutex tester */ #define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */ -#define PF_WAKE_UP_IDLE 0x80000000 /* try to wake up on an idle CPU */ +#define PF_SUSPEND_TASK 0x80000000 /* this thread called freeze_processes and should not be frozen */ /* * Only the _current_ task can read/write to tsk->flags, but other @@ -1862,17 +2138,31 @@ static inline void memalloc_noio_restore(unsigned int flags) } /* Per-process atomic flags. */ -#define PFA_NO_NEW_PRIVS 0x00000001 /* May not gain new privileges. */ +#define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */ +#define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */ +#define PFA_SPREAD_SLAB 2 /* Spread some slab caches over cpuset */ -static inline bool task_no_new_privs(struct task_struct *p) -{ - return test_bit(PFA_NO_NEW_PRIVS, &p->atomic_flags); -} -static inline void task_set_no_new_privs(struct task_struct *p) -{ - set_bit(PFA_NO_NEW_PRIVS, &p->atomic_flags); -} +#define TASK_PFA_TEST(name, func) \ + static inline bool task_##func(struct task_struct *p) \ + { return test_bit(PFA_##name, &p->atomic_flags); } +#define TASK_PFA_SET(name, func) \ + static inline void task_set_##func(struct task_struct *p) \ + { set_bit(PFA_##name, &p->atomic_flags); } +#define TASK_PFA_CLEAR(name, func) \ + static inline void task_clear_##func(struct task_struct *p) \ + { clear_bit(PFA_##name, &p->atomic_flags); } + +TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs) +TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs) + +TASK_PFA_TEST(SPREAD_PAGE, spread_page) +TASK_PFA_SET(SPREAD_PAGE, spread_page) +TASK_PFA_CLEAR(SPREAD_PAGE, spread_page) + +TASK_PFA_TEST(SPREAD_SLAB, spread_slab) +TASK_PFA_SET(SPREAD_SLAB, spread_slab) +TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab) /* * task->jobctl flags @@ -1903,33 +2193,24 @@ extern bool task_set_jobctl_pending(struct task_struct *task, extern void task_clear_jobctl_trapping(struct task_struct *task); extern void task_clear_jobctl_pending(struct task_struct *task, unsigned int mask); - -#ifdef CONFIG_PREEMPT_RCU - #define RCU_READ_UNLOCK_BLOCKED (1 << 0) /* blocked while in RCU read-side. */ #define RCU_READ_UNLOCK_NEED_QS (1 << 1) /* RCU core needs CPU response. */ static inline void rcu_copy_process(struct task_struct *p) { +#ifdef CONFIG_PREEMPT_RCU p->rcu_read_lock_nesting = 0; p->rcu_read_unlock_special = 0; -#ifdef CONFIG_TREE_PREEMPT_RCU p->rcu_blocked_node = NULL; -#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */ -#ifdef CONFIG_RCU_BOOST - p->rcu_boost_mutex = NULL; -#endif /* #ifdef CONFIG_RCU_BOOST */ INIT_LIST_HEAD(&p->rcu_node_entry); +#endif /* #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_TASKS_RCU + p->rcu_tasks_holdout = false; + INIT_LIST_HEAD(&p->rcu_tasks_holdout_list); + p->rcu_tasks_idle_cpu = -1; +#endif /* #ifdef CONFIG_TASKS_RCU */ } -#else - -static inline void rcu_copy_process(struct task_struct *p) -{ -} - -#endif - static inline void tsk_restore_flags(struct task_struct *task, unsigned long orig_flags, unsigned long flags) { @@ -1945,6 +2226,8 @@ extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask); extern void sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency); +extern void sched_set_cluster_dstate(const cpumask_t *cluster_cpus, int dstate, + int wakeup_energy, int wakeup_latency); #else static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) @@ -1961,6 +2244,11 @@ static inline void sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency) { } + +static inline void sched_set_cluster_dstate(const cpumask_t *cluster_cpus, + int dstate, int wakeup_energy, int wakeup_latency) +{ +} #endif extern int sched_set_wake_up_idle(struct task_struct *p, int wake_up_idle); @@ -1971,6 +2259,11 @@ extern u32 sched_get_wake_up_idle(struct task_struct *p); extern int sched_set_boost(int enable); extern int sched_set_init_task_load(struct task_struct *p, int init_load_pct); extern u32 sched_get_init_task_load(struct task_struct *p); +extern int sched_set_static_cpu_pwr_cost(int cpu, unsigned int cost); +extern unsigned int sched_get_static_cpu_pwr_cost(int cpu); +extern int sched_set_static_cluster_pwr_cost(int cpu, unsigned int cost); +extern unsigned int sched_get_static_cluster_pwr_cost(int cpu); +#ifdef CONFIG_SCHED_QHMP extern int sched_set_cpu_prefer_idle(int cpu, int prefer_idle); extern int sched_get_cpu_prefer_idle(int cpu); extern int sched_set_cpu_mostly_idle_load(int cpu, int mostly_idle_pct); @@ -1980,6 +2273,7 @@ extern int sched_get_cpu_mostly_idle_nr_run(int cpu); extern int sched_set_cpu_mostly_idle_freq(int cpu, unsigned int mostly_idle_freq); extern unsigned int sched_get_cpu_mostly_idle_freq(int cpu); +#endif #else static inline int sched_set_boost(int enable) @@ -2050,7 +2344,9 @@ static inline void sched_clock_idle_wakeup_event(u64 delta_ns) * but then during bootup it turns out that sched_clock() * is reliable after all: */ -extern int sched_clock_stable; +extern int sched_clock_stable(void); +extern void set_sched_clock_stable(void); +extern void clear_sched_clock_stable(void); extern void sched_clock_tick(void); extern void sched_clock_idle_sleep_event(void); @@ -2118,10 +2414,19 @@ static inline void sched_autogroup_fork(struct signal_struct *sig) { } static inline void sched_autogroup_exit(struct signal_struct *sig) { } #endif -extern bool yield_to(struct task_struct *p, bool preempt); +extern int yield_to(struct task_struct *p, bool preempt); extern void set_user_nice(struct task_struct *p, long nice); extern int task_prio(const struct task_struct *p); -extern int task_nice(const struct task_struct *p); +/** + * task_nice - return the nice value of a given task. + * @p: the task in question. + * + * Return: The nice value [ -20 ... 0 ... 19 ]. + */ +static inline int task_nice(const struct task_struct *p) +{ + return PRIO_TO_NICE((p)->static_prio); +} extern int can_nice(const struct task_struct *p, const int nice); extern int task_curr(const struct task_struct *p); extern int idle_cpu(int cpu); @@ -2135,6 +2440,8 @@ extern struct task_struct *idle_task(int cpu); /** * is_idle_task - is the specified task an idle task? * @p: the task in question. + * + * Return: 1 if @p is an idle task. 0 otherwise. */ static inline bool is_idle_task(const struct task_struct *p) { @@ -2188,7 +2495,6 @@ extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns); extern void __set_special_pids(struct pid *pid); - /* per-UID process charging. */ extern struct user_struct * alloc_uid(kuid_t); static inline struct user_struct *get_uid(struct user_struct *u) @@ -2210,7 +2516,7 @@ extern void wake_up_new_task(struct task_struct *tsk); #else static inline void kick_process(struct task_struct *tsk) { } #endif -extern void sched_fork(unsigned long clone_flags, struct task_struct *p); +extern int sched_fork(unsigned long clone_flags, struct task_struct *p); extern void sched_dead(struct task_struct *p); #ifdef CONFIG_SCHED_HMP extern void sched_exit(struct task_struct *p); @@ -2301,8 +2607,10 @@ static inline int on_sig_stack(unsigned long sp) static inline int sas_ss_flags(unsigned long sp) { - return (current->sas_ss_size == 0 ? SS_DISABLE - : on_sig_stack(sp) ? SS_ONSTACK : 0); + if (!current->sas_ss_size) + return SS_DISABLE; + + return on_sig_stack(sp) ? SS_ONSTACK : 0; } static inline unsigned long sigsp(unsigned long sp, struct ksignal *ksig) @@ -2341,8 +2649,6 @@ extern struct mm_struct *get_task_mm(struct task_struct *task); extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode); /* Remove the current tasks stale references to the old mm_struct */ extern void mm_release(struct task_struct *, struct mm_struct *); -/* Allocate a new mm structure and copy contents from tsk->mm */ -extern struct mm_struct *dup_mm(struct task_struct *tsk); extern int copy_thread(unsigned long, unsigned long, unsigned long, struct task_struct *); @@ -2367,6 +2673,7 @@ extern long do_fork(unsigned long, unsigned long, unsigned long, int __user *, i struct task_struct *fork_idle(int); extern pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags); +extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec); extern void set_task_comm(struct task_struct *tsk, char *from); extern char *get_task_comm(char *to, struct task_struct *tsk); @@ -2549,12 +2856,27 @@ static inline void setup_thread_stack(struct task_struct *p, struct task_struct task_thread_info(p)->task = p; } +/* + * Return the address of the last usable long on the stack. + * + * When the stack grows down, this is just above the thread + * info struct. Going any lower will corrupt the threadinfo. + * + * When the stack grows up, this is the highest address. + * Beyond that position, we corrupt data on the next page. + */ static inline unsigned long *end_of_stack(struct task_struct *p) { +#ifdef CONFIG_STACK_GROWSUP + return (unsigned long *)((unsigned long)task_thread_info(p) + THREAD_SIZE) - 1; +#else return (unsigned long *)(task_thread_info(p) + 1); +#endif } #endif +#define task_stack_end_corrupted(task) \ + (*(end_of_stack(task)) != STACK_END_MAGIC) static inline int object_is_on_stack(void *obj) { @@ -2577,6 +2899,7 @@ static inline unsigned long stack_not_used(struct task_struct *p) return (unsigned long)n - (unsigned long)end_of_stack(p); } #endif +extern void set_task_stack_end_magic(struct task_struct *tsk); /* set thread flags in other task's structures * - see asm/thread_info.h for TIF_xxxx flags available @@ -2652,11 +2975,6 @@ static inline int signal_pending_state(long state, struct task_struct *p) return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p); } -static inline int need_resched(void) -{ - return unlikely(test_thread_flag(TIF_NEED_RESCHED)); -} - /* * cond_resched() and cond_resched_lock(): latency reduction via * explicit rescheduling in places that are safe. The return @@ -2691,6 +3009,15 @@ extern int __cond_resched_softirq(void); __cond_resched_softirq(); \ }) +static inline void cond_resched_rcu(void) +{ +#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU) + rcu_read_unlock(); + cond_resched(); + rcu_read_lock(); +#endif +} + /* * Does a critical section need to be broken due to another * task waiting?: (technically does not depend on CONFIG_PREEMPT, @@ -2707,51 +3034,9 @@ static inline int spin_needbreak(spinlock_t *lock) /* * Idle thread specific functions to determine the need_resched - * polling state. We have two versions, one based on TS_POLLING in - * thread_info.status and one based on TIF_POLLING_NRFLAG in - * thread_info.flags + * polling state. */ -#ifdef TS_POLLING -static inline int tsk_is_polling(struct task_struct *p) -{ - return task_thread_info(p)->status & TS_POLLING; -} -static inline void __current_set_polling(void) -{ - current_thread_info()->status |= TS_POLLING; -} - -static inline bool __must_check current_set_polling_and_test(void) -{ - __current_set_polling(); - - /* - * Polling state must be visible before we test NEED_RESCHED, - * paired by resched_task() - */ - smp_mb(); - - return unlikely(tif_need_resched()); -} - -static inline void __current_clr_polling(void) -{ - current_thread_info()->status &= ~TS_POLLING; -} - -static inline bool __must_check current_clr_polling_and_test(void) -{ - __current_clr_polling(); - - /* - * Polling state must be visible before we test NEED_RESCHED, - * paired by resched_task() - */ - smp_mb(); - - return unlikely(tif_need_resched()); -} -#elif defined(TIF_POLLING_NRFLAG) +#ifdef TIF_POLLING_NRFLAG static inline int tsk_is_polling(struct task_struct *p) { return test_tsk_thread_flag(p, TIF_POLLING_NRFLAG); @@ -2768,7 +3053,7 @@ static inline bool __must_check current_set_polling_and_test(void) /* * Polling state must be visible before we test NEED_RESCHED, - * paired by resched_task() + * paired by resched_curr() */ smp_mb__after_atomic(); @@ -2786,7 +3071,7 @@ static inline bool __must_check current_clr_polling_and_test(void) /* * Polling state must be visible before we test NEED_RESCHED, - * paired by resched_task() + * paired by resched_curr() */ smp_mb__after_atomic(); @@ -2808,6 +3093,26 @@ static inline bool __must_check current_clr_polling_and_test(void) } #endif +static inline void current_clr_polling(void) +{ + __current_clr_polling(); + + /* + * Ensure we check TIF_NEED_RESCHED after we clear the polling bit. + * Once the bit is cleared, we'll get IPIs with every new + * TIF_NEED_RESCHED and the IPI handler, scheduler_ipi(), will also + * fold. + */ + smp_mb(); /* paired with resched_curr() */ + + preempt_fold_need_resched(); +} + +static __always_inline bool need_resched(void) +{ + return unlikely(tif_need_resched()); +} + /* * Thread group CPU time accounting. */ @@ -2849,6 +3154,11 @@ static inline unsigned int task_cpu(const struct task_struct *p) return task_thread_info(p)->cpu; } +static inline int task_node(const struct task_struct *p) +{ + return cpu_to_node(task_cpu(p)); +} + extern void set_task_cpu(struct task_struct *p, unsigned int cpu); #else @@ -2932,11 +3242,8 @@ extern void mm_init_owner(struct mm_struct *mm, struct task_struct *p); static inline void mm_update_next_owner(struct mm_struct *mm) { } - -static inline void mm_init_owner(struct mm_struct *mm, struct task_struct *p) -{ -} -#endif /* CONFIG_MM_OWNER */ +static inline void mm_init_owner(struct mm_struct *mm, struct task_struct *p) {} +#endif /* CONFIG_MEMCG */ static inline unsigned long task_rlimit(const struct task_struct *tsk, unsigned int limit) diff --git a/include/linux/sched/deadline.h b/include/linux/sched/deadline.h new file mode 100644 index 0000000000000..9d303b8847df2 --- /dev/null +++ b/include/linux/sched/deadline.h @@ -0,0 +1,24 @@ +#ifndef _SCHED_DEADLINE_H +#define _SCHED_DEADLINE_H + +/* + * SCHED_DEADLINE tasks has negative priorities, reflecting + * the fact that any of them has higher prio than RT and + * NORMAL/BATCH tasks. + */ + +#define MAX_DL_PRIO 0 + +static inline int dl_prio(int prio) +{ + if (unlikely(prio < MAX_DL_PRIO)) + return 1; + return 0; +} + +static inline int dl_task(struct task_struct *p) +{ + return dl_prio(p->prio); +} + +#endif /* _SCHED_DEADLINE_H */ diff --git a/include/linux/sched/prio.h b/include/linux/sched/prio.h new file mode 100644 index 0000000000000..d9cf5a5762d9d --- /dev/null +++ b/include/linux/sched/prio.h @@ -0,0 +1,60 @@ +#ifndef _SCHED_PRIO_H +#define _SCHED_PRIO_H + +#define MAX_NICE 19 +#define MIN_NICE -20 +#define NICE_WIDTH (MAX_NICE - MIN_NICE + 1) + +/* + * Priority of a process goes from 0..MAX_PRIO-1, valid RT + * priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL/SCHED_BATCH + * tasks are in the range MAX_RT_PRIO..MAX_PRIO-1. Priority + * values are inverted: lower p->prio value means higher priority. + * + * The MAX_USER_RT_PRIO value allows the actual maximum + * RT priority to be separate from the value exported to + * user-space. This allows kernel threads to set their + * priority to a value higher than any user task. Note: + * MAX_RT_PRIO must not be smaller than MAX_USER_RT_PRIO. + */ + +#define MAX_USER_RT_PRIO 100 +#define MAX_RT_PRIO MAX_USER_RT_PRIO + +#define MAX_PRIO (MAX_RT_PRIO + NICE_WIDTH) +#define DEFAULT_PRIO (MAX_RT_PRIO + NICE_WIDTH / 2) + +/* + * Convert user-nice values [ -20 ... 0 ... 19 ] + * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ], + * and back. + */ +#define NICE_TO_PRIO(nice) ((nice) + DEFAULT_PRIO) +#define PRIO_TO_NICE(prio) ((prio) - DEFAULT_PRIO) + +/* + * 'User priority' is the nice value converted to something we + * can work with better when scaling various scheduler parameters, + * it's a [ 0 ... 39 ] range. + */ +#define USER_PRIO(p) ((p)-MAX_RT_PRIO) +#define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio) +#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO)) + +/* + * Convert nice value [19,-20] to rlimit style value [1,40]. + */ +static inline long nice_to_rlimit(long nice) +{ + return (MAX_NICE - nice + 1); +} + +/* + * Convert rlimit style value [1,40] to nice value [-20, 19]. + */ +static inline long rlimit_to_nice(long prio) +{ + return (MAX_NICE - prio + 1); +} + +#endif /* _SCHED_PRIO_H */ diff --git a/include/linux/sched/rt.h b/include/linux/sched/rt.h index 440434df36274..5b85463773964 100644 --- a/include/linux/sched/rt.h +++ b/include/linux/sched/rt.h @@ -1,24 +1,7 @@ #ifndef _SCHED_RT_H #define _SCHED_RT_H -/* - * Priority of a process goes from 0..MAX_PRIO-1, valid RT - * priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL/SCHED_BATCH - * tasks are in the range MAX_RT_PRIO..MAX_PRIO-1. Priority - * values are inverted: lower p->prio value means higher priority. - * - * The MAX_USER_RT_PRIO value allows the actual maximum - * RT priority to be separate from the value exported to - * user-space. This allows kernel threads to set their - * priority to a value higher than any user task. Note: - * MAX_RT_PRIO must not be smaller than MAX_USER_RT_PRIO. - */ - -#define MAX_USER_RT_PRIO 100 -#define MAX_RT_PRIO MAX_USER_RT_PRIO - -#define MAX_PRIO (MAX_RT_PRIO + 40) -#define DEFAULT_PRIO (MAX_RT_PRIO + 20) +#include static inline int rt_prio(int prio) { @@ -34,7 +17,19 @@ static inline int rt_task(struct task_struct *p) #ifdef CONFIG_RT_MUTEXES extern int rt_mutex_getprio(struct task_struct *p); +extern int rt_mutex_getprio(struct task_struct *p); + extern void rt_mutex_setprio(struct task_struct *p, int prio); +extern int rt_mutex_get_effective_prio(struct task_struct *task, int newprio); +#if 0 +extern struct task_struct *rt_mutex_get_top_task(struct task_struct *task); +#else +// TJK: hack since CONFIG_RT_MUTEXES gets turned on implicitely +static inline struct task_struct *rt_mutex_get_top_task(struct task_struct *task) +{ + return NULL; +} +#endif extern void rt_mutex_adjust_pi(struct task_struct *p); static inline bool tsk_is_pi_blocked(struct task_struct *tsk) { @@ -45,6 +40,17 @@ static inline int rt_mutex_getprio(struct task_struct *p) { return p->normal_prio; } + +static inline int rt_mutex_get_effective_prio(struct task_struct *task, + int newprio) +{ + return newprio; +} + +static inline struct task_struct *rt_mutex_get_top_task(struct task_struct *task) +{ + return NULL; +} # define rt_mutex_adjust_pi(p) do { } while (0) static inline bool tsk_is_pi_blocked(struct task_struct *tsk) { diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 4d892dc630f75..e99174dd0afa0 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -2,10 +2,10 @@ #define _SCHED_SYSCTL_H #ifdef CONFIG_DETECT_HUNG_TASK +extern int sysctl_hung_task_check_count; extern unsigned int sysctl_hung_task_panic; -extern unsigned long sysctl_hung_task_check_count; extern unsigned long sysctl_hung_task_timeout_secs; -extern unsigned long sysctl_hung_task_warnings; +extern int sysctl_hung_task_warnings; extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); @@ -25,6 +25,10 @@ enum { sysctl_hung_task_timeout_secs = 0 }; * Because the kernel adds some informative sections to a image of program at * generating coredump, we need some margin. The number of extra sections is * 1-3 now and depends on arch. We use "5" as safe margin, here. + * + * ELF extended numbering allows more than 65535 sections, so 16-bit bound is + * not a hard limit any more. Although some userspace tools can be surprised by + * that. */ #define MAPCOUNT_ELF_CORE_MARGIN (5) #define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN) @@ -44,7 +48,6 @@ extern unsigned int sysctl_sched_cpu_high_irqload; extern unsigned int sysctl_sched_freq_account_wait_time; extern unsigned int sysctl_sched_migration_fixup; extern unsigned int sysctl_sched_heavy_task_pct; -extern unsigned int sysctl_sched_min_runtime; extern unsigned int sysctl_sched_enable_power_aware; #ifdef VENDOR_EDIT extern unsigned int sysctl_thermal_aware_scheduling; @@ -62,12 +65,24 @@ extern int sysctl_sched_freq_dec_notify; #ifdef CONFIG_SCHED_HMP extern unsigned int sysctl_sched_spill_nr_run; extern unsigned int sysctl_sched_spill_load_pct; -extern unsigned int sysctl_sched_small_task_pct; extern unsigned int sysctl_sched_upmigrate_pct; extern unsigned int sysctl_sched_downmigrate_pct; extern int sysctl_sched_upmigrate_min_nice; extern unsigned int sysctl_sched_powerband_limit_pct; extern unsigned int sysctl_sched_boost; +extern unsigned int sysctl_power_aware_timer_migration; +extern unsigned int sysctl_early_detection_duration; + +#ifdef CONFIG_SCHED_QHMP +extern unsigned int sysctl_sched_min_runtime; +extern unsigned int sysctl_sched_small_task_pct; +#else +extern unsigned int sysctl_sched_lowspill_freq; +extern unsigned int sysctl_sched_pack_freq; +#if defined(CONFIG_SCHED_FREQ_INPUT) +extern unsigned int sysctl_sched_new_task_windows; +#endif +#endif #else /* CONFIG_SCHED_HMP */ @@ -86,9 +101,7 @@ extern enum sched_tunable_scaling sysctl_sched_tunable_scaling; extern unsigned int sysctl_numa_balancing_scan_delay; extern unsigned int sysctl_numa_balancing_scan_period_min; extern unsigned int sysctl_numa_balancing_scan_period_max; -extern unsigned int sysctl_numa_balancing_scan_period_reset; extern unsigned int sysctl_numa_balancing_scan_size; -extern unsigned int sysctl_numa_balancing_settle_count; #ifdef CONFIG_SCHED_DEBUG extern unsigned int sysctl_sched_migration_cost; @@ -143,6 +156,15 @@ extern unsigned int sysctl_sched_cfs_bandwidth_slice; extern unsigned int sysctl_sched_autogroup_enabled; #endif +#ifdef CONFIG_SCHEDSTATS +extern unsigned int sysctl_sched_latency_panic_threshold; +extern unsigned int sysctl_sched_latency_warn_threshold; + +extern int sched_max_latency_sysctl(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos); +#endif + extern int sched_rr_timeslice; extern int sched_rr_handler(struct ctl_table *table, int write, @@ -153,4 +175,8 @@ extern int sched_rt_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); +extern int sysctl_numa_balancing(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos); + #endif /* _SCHED_SYSCTL_H */ diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 383f8231e4367..7be0071adc149 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -70,6 +70,7 @@ void user_enter(void) local_irq_restore(flags); } +#ifdef TJK_HMP #ifdef CONFIG_PREEMPT /** * preempt_schedule_context - preempt_schedule called by tracing @@ -110,6 +111,7 @@ void __sched notrace preempt_schedule_context(void) } EXPORT_SYMBOL_GPL(preempt_schedule_context); #endif /* CONFIG_PREEMPT */ +#endif /** * user_exit - Inform the context tracking that the CPU is diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 0f4e25cc07ac7..ea6e4df4f860e 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -11,9 +11,13 @@ ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer endif -obj-y += core.o clock.o cputime.o idle_task.o fair.o rt.o stop_task.o sched_avg.o energy.o -obj-$(CONFIG_SMP) += cpupri.o +obj-y += core.o proc.o clock.o cputime.o +obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o energy.o +obj-y += completion.o +obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o obj-$(CONFIG_SCHEDSTATS) += stats.o obj-$(CONFIG_SCHED_DEBUG) += debug.o +obj-$(CONFIG_SCHED_TUNE) += tune.o obj-$(CONFIG_CGROUP_CPUACCT) += cpuacct.o +obj-$(CONFIG_CPU_FREQ_GOV_SCHED) += cpufreq_sched.o diff --git a/kernel/sched/auto_group.c b/kernel/sched/auto_group.c index 4a073539c58e6..eae160dd669d9 100644 --- a/kernel/sched/auto_group.c +++ b/kernel/sched/auto_group.c @@ -87,8 +87,7 @@ static inline struct autogroup *autogroup_create(void) * so we don't have to move tasks around upon policy change, * or flail around trying to allocate bandwidth on the fly. * A bandwidth exception in __sched_setscheduler() allows - * the policy change to proceed. Thereafter, task_group() - * returns &root_task_group, so zero bandwidth is required. + * the policy change to proceed. */ free_rt_sched_group(tg); tg->rt_se = root_task_group.rt_se; @@ -115,9 +114,6 @@ bool task_wants_autogroup(struct task_struct *p, struct task_group *tg) if (tg != &root_task_group) return false; - if (p->sched_class != &fair_sched_class) - return false; - /* * We can only assume the task group can't go away on us if * autogroup_move_group() can see us on ->thread_group list. @@ -148,11 +144,8 @@ autogroup_move_group(struct task_struct *p, struct autogroup *ag) if (!ACCESS_ONCE(sysctl_sched_autogroup_enabled)) goto out; - t = p; - do { + for_each_thread(p, t) sched_move_task(t); - } while_each_thread(p, t); - out: unlock_task_sighand(p, &flags); autogroup_kref_put(prev); @@ -203,7 +196,7 @@ int proc_sched_autogroup_set_nice(struct task_struct *p, int nice) struct autogroup *ag; int err; - if (nice < -20 || nice > 19) + if (nice < MIN_NICE || nice > MAX_NICE) return -EINVAL; err = security_task_setnice(current, nice); diff --git a/kernel/sched/clock.c b/kernel/sched/clock.c index c3ae1446461c0..c27e4f8f4879e 100644 --- a/kernel/sched/clock.c +++ b/kernel/sched/clock.c @@ -26,9 +26,10 @@ * at 0 on boot (but people really shouldn't rely on that). * * cpu_clock(i) -- can be used from any context, including NMI. - * sched_clock_cpu(i) -- must be used with local IRQs disabled (implied by NMI) * local_clock() -- is cpu_clock() on the current cpu. * + * sched_clock_cpu(i) + * * How: * * The implementation either uses sched_clock() when @@ -50,15 +51,6 @@ * Furthermore, explicit sleep and wakeup hooks allow us to account for time * that is otherwise invisible (TSC gets stopped). * - * - * Notes: - * - * The !IRQ-safetly of sched_clock() and sched_clock_cpu() comes from things - * like cpufreq interrupts that can change the base clock (TSC) multiplier - * and cause funny jumps in time -- although the filtering provided by - * sched_clock_cpu() should mitigate serious artifacts we cannot rely on it - * in general since for !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK we fully rely on - * sched_clock(). */ #include #include @@ -66,13 +58,16 @@ #include #include #include +#include +#include +#include /* * Scheduler clock - returns current time in nanosec units. * This is default implementation. * Architectures and sub-architectures can override this. */ -unsigned long long __attribute__((weak)) sched_clock(void) +unsigned long long __weak sched_clock(void) { return (unsigned long long)(jiffies - INITIAL_JIFFIES) * (NSEC_PER_SEC / HZ); @@ -82,7 +77,52 @@ EXPORT_SYMBOL_GPL(sched_clock); __read_mostly int sched_clock_running; #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK -__read_mostly int sched_clock_stable; +static struct static_key __sched_clock_stable = STATIC_KEY_INIT; +static int __sched_clock_stable_early; + +int sched_clock_stable(void) +{ + return static_key_false(&__sched_clock_stable); +} + +static void __set_sched_clock_stable(void) +{ + if (!sched_clock_stable()) + static_key_slow_inc(&__sched_clock_stable); +} + +void set_sched_clock_stable(void) +{ + __sched_clock_stable_early = 1; + + smp_mb(); /* matches sched_clock_init() */ + + if (!sched_clock_running) + return; + + __set_sched_clock_stable(); +} + +static void __clear_sched_clock_stable(struct work_struct *work) +{ + /* XXX worry about clock continuity */ + if (sched_clock_stable()) + static_key_slow_dec(&__sched_clock_stable); +} + +static DECLARE_WORK(sched_clock_work, __clear_sched_clock_stable); + +void clear_sched_clock_stable(void) +{ + __sched_clock_stable_early = 0; + + smp_mb(); /* matches sched_clock_init() */ + + if (!sched_clock_running) + return; + + schedule_work(&sched_clock_work); +} struct sched_clock_data { u64 tick_raw; @@ -94,7 +134,7 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); static inline struct sched_clock_data *this_scd(void) { - return &__get_cpu_var(sched_clock_data); + return this_cpu_ptr(&sched_clock_data); } static inline struct sched_clock_data *cpu_sdc(int cpu) @@ -116,6 +156,20 @@ void sched_clock_init(void) } sched_clock_running = 1; + + /* + * Ensure that it is impossible to not do a static_key update. + * + * Either {set,clear}_sched_clock_stable() must see sched_clock_running + * and do the update, or we must see their __sched_clock_stable_early + * and do the update, or both. + */ + smp_mb(); /* matches {set,clear}_sched_clock_stable() */ + + if (__sched_clock_stable_early) + __set_sched_clock_stable(); + else + __clear_sched_clock_stable(NULL); } /* @@ -242,20 +296,20 @@ u64 sched_clock_cpu(int cpu) struct sched_clock_data *scd; u64 clock; - WARN_ON_ONCE(!irqs_disabled()); - - if (sched_clock_stable) + if (sched_clock_stable()) return sched_clock(); if (unlikely(!sched_clock_running)) return 0ull; + preempt_disable_notrace(); scd = cpu_sdc(cpu); if (cpu != smp_processor_id()) clock = sched_clock_remote(scd); else clock = sched_clock_local(scd); + preempt_enable_notrace(); return clock; } @@ -265,7 +319,7 @@ void sched_clock_tick(void) struct sched_clock_data *scd; u64 now, now_gtod; - if (sched_clock_stable) + if (sched_clock_stable()) return; if (unlikely(!sched_clock_running)) @@ -316,14 +370,10 @@ EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); */ u64 cpu_clock(int cpu) { - u64 clock; - unsigned long flags; - - local_irq_save(flags); - clock = sched_clock_cpu(cpu); - local_irq_restore(flags); + if (!sched_clock_stable()) + return sched_clock_cpu(cpu); - return clock; + return sched_clock(); } /* @@ -335,14 +385,10 @@ u64 cpu_clock(int cpu) */ u64 local_clock(void) { - u64 clock; - unsigned long flags; + if (!sched_clock_stable()) + return sched_clock_cpu(raw_smp_processor_id()); - local_irq_save(flags); - clock = sched_clock_cpu(smp_processor_id()); - local_irq_restore(flags); - - return clock; + return sched_clock(); } #else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ @@ -362,12 +408,12 @@ u64 sched_clock_cpu(int cpu) u64 cpu_clock(int cpu) { - return sched_clock_cpu(cpu); + return sched_clock(); } u64 local_clock(void) { - return sched_clock_cpu(0); + return sched_clock(); } #endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ diff --git a/kernel/sched/completion.c b/kernel/sched/completion.c new file mode 100644 index 0000000000000..a63f4dc27909e --- /dev/null +++ b/kernel/sched/completion.c @@ -0,0 +1,299 @@ +/* + * Generic wait-for-completion handler; + * + * It differs from semaphores in that their default case is the opposite, + * wait_for_completion default blocks whereas semaphore default non-block. The + * interface also makes it easy to 'complete' multiple waiting threads, + * something which isn't entirely natural for semaphores. + * + * But more importantly, the primitive documents the usage. Semaphores would + * typically be used for exclusion which gives rise to priority inversion. + * Waiting for completion is a typically sync point, but not an exclusion point. + */ + +#include +#include + +/** + * complete: - signals a single thread waiting on this completion + * @x: holds the state of this particular completion + * + * This will wake up a single thread waiting on this completion. Threads will be + * awakened in the same order in which they were queued. + * + * See also complete_all(), wait_for_completion() and related routines. + * + * It may be assumed that this function implies a write memory barrier before + * changing the task state if and only if any tasks are woken up. + */ +void complete(struct completion *x) +{ + unsigned long flags; + + spin_lock_irqsave(&x->wait.lock, flags); + x->done++; + __wake_up_locked(&x->wait, TASK_NORMAL, 1); + spin_unlock_irqrestore(&x->wait.lock, flags); +} +EXPORT_SYMBOL(complete); + +/** + * complete_all: - signals all threads waiting on this completion + * @x: holds the state of this particular completion + * + * This will wake up all threads waiting on this particular completion event. + * + * It may be assumed that this function implies a write memory barrier before + * changing the task state if and only if any tasks are woken up. + */ +void complete_all(struct completion *x) +{ + unsigned long flags; + + spin_lock_irqsave(&x->wait.lock, flags); + x->done += UINT_MAX/2; + __wake_up_locked(&x->wait, TASK_NORMAL, 0); + spin_unlock_irqrestore(&x->wait.lock, flags); +} +EXPORT_SYMBOL(complete_all); + +static inline long __sched +do_wait_for_common(struct completion *x, + long (*action)(long), long timeout, int state) +{ + if (!x->done) { + DECLARE_WAITQUEUE(wait, current); + + __add_wait_queue_tail_exclusive(&x->wait, &wait); + do { + if (signal_pending_state(state, current)) { + timeout = -ERESTARTSYS; + break; + } + __set_current_state(state); + spin_unlock_irq(&x->wait.lock); + timeout = action(timeout); + spin_lock_irq(&x->wait.lock); + } while (!x->done && timeout); + __remove_wait_queue(&x->wait, &wait); + if (!x->done) + return timeout; + } + x->done--; + return timeout ?: 1; +} + +static inline long __sched +__wait_for_common(struct completion *x, + long (*action)(long), long timeout, int state) +{ + might_sleep(); + + spin_lock_irq(&x->wait.lock); + timeout = do_wait_for_common(x, action, timeout, state); + spin_unlock_irq(&x->wait.lock); + return timeout; +} + +static long __sched +wait_for_common(struct completion *x, long timeout, int state) +{ + return __wait_for_common(x, schedule_timeout, timeout, state); +} + +static long __sched +wait_for_common_io(struct completion *x, long timeout, int state) +{ + return __wait_for_common(x, io_schedule_timeout, timeout, state); +} + +/** + * wait_for_completion: - waits for completion of a task + * @x: holds the state of this particular completion + * + * This waits to be signaled for completion of a specific task. It is NOT + * interruptible and there is no timeout. + * + * See also similar routines (i.e. wait_for_completion_timeout()) with timeout + * and interrupt capability. Also see complete(). + */ +void __sched wait_for_completion(struct completion *x) +{ + wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); +} +EXPORT_SYMBOL(wait_for_completion); + +/** + * wait_for_completion_timeout: - waits for completion of a task (w/timeout) + * @x: holds the state of this particular completion + * @timeout: timeout value in jiffies + * + * This waits for either a completion of a specific task to be signaled or for a + * specified timeout to expire. The timeout is in jiffies. It is not + * interruptible. + * + * Return: 0 if timed out, and positive (at least 1, or number of jiffies left + * till timeout) if completed. + */ +unsigned long __sched +wait_for_completion_timeout(struct completion *x, unsigned long timeout) +{ + return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE); +} +EXPORT_SYMBOL(wait_for_completion_timeout); + +/** + * wait_for_completion_io: - waits for completion of a task + * @x: holds the state of this particular completion + * + * This waits to be signaled for completion of a specific task. It is NOT + * interruptible and there is no timeout. The caller is accounted as waiting + * for IO. + */ +void __sched wait_for_completion_io(struct completion *x) +{ + wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); +} +EXPORT_SYMBOL(wait_for_completion_io); + +/** + * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout) + * @x: holds the state of this particular completion + * @timeout: timeout value in jiffies + * + * This waits for either a completion of a specific task to be signaled or for a + * specified timeout to expire. The timeout is in jiffies. It is not + * interruptible. The caller is accounted as waiting for IO. + * + * Return: 0 if timed out, and positive (at least 1, or number of jiffies left + * till timeout) if completed. + */ +unsigned long __sched +wait_for_completion_io_timeout(struct completion *x, unsigned long timeout) +{ + return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE); +} +EXPORT_SYMBOL(wait_for_completion_io_timeout); + +/** + * wait_for_completion_interruptible: - waits for completion of a task (w/intr) + * @x: holds the state of this particular completion + * + * This waits for completion of a specific task to be signaled. It is + * interruptible. + * + * Return: -ERESTARTSYS if interrupted, 0 if completed. + */ +int __sched wait_for_completion_interruptible(struct completion *x) +{ + long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE); + if (t == -ERESTARTSYS) + return t; + return 0; +} +EXPORT_SYMBOL(wait_for_completion_interruptible); + +/** + * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr)) + * @x: holds the state of this particular completion + * @timeout: timeout value in jiffies + * + * This waits for either a completion of a specific task to be signaled or for a + * specified timeout to expire. It is interruptible. The timeout is in jiffies. + * + * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1, + * or number of jiffies left till timeout) if completed. + */ +long __sched +wait_for_completion_interruptible_timeout(struct completion *x, + unsigned long timeout) +{ + return wait_for_common(x, timeout, TASK_INTERRUPTIBLE); +} +EXPORT_SYMBOL(wait_for_completion_interruptible_timeout); + +/** + * wait_for_completion_killable: - waits for completion of a task (killable) + * @x: holds the state of this particular completion + * + * This waits to be signaled for completion of a specific task. It can be + * interrupted by a kill signal. + * + * Return: -ERESTARTSYS if interrupted, 0 if completed. + */ +int __sched wait_for_completion_killable(struct completion *x) +{ + long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE); + if (t == -ERESTARTSYS) + return t; + return 0; +} +EXPORT_SYMBOL(wait_for_completion_killable); + +/** + * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable)) + * @x: holds the state of this particular completion + * @timeout: timeout value in jiffies + * + * This waits for either a completion of a specific task to be + * signaled or for a specified timeout to expire. It can be + * interrupted by a kill signal. The timeout is in jiffies. + * + * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1, + * or number of jiffies left till timeout) if completed. + */ +long __sched +wait_for_completion_killable_timeout(struct completion *x, + unsigned long timeout) +{ + return wait_for_common(x, timeout, TASK_KILLABLE); +} +EXPORT_SYMBOL(wait_for_completion_killable_timeout); + +/** + * try_wait_for_completion - try to decrement a completion without blocking + * @x: completion structure + * + * Return: 0 if a decrement cannot be done without blocking + * 1 if a decrement succeeded. + * + * If a completion is being used as a counting completion, + * attempt to decrement the counter without blocking. This + * enables us to avoid waiting if the resource the completion + * is protecting is not available. + */ +bool try_wait_for_completion(struct completion *x) +{ + unsigned long flags; + int ret = 1; + + spin_lock_irqsave(&x->wait.lock, flags); + if (!x->done) + ret = 0; + else + x->done--; + spin_unlock_irqrestore(&x->wait.lock, flags); + return ret; +} +EXPORT_SYMBOL(try_wait_for_completion); + +/** + * completion_done - Test to see if a completion has any waiters + * @x: completion structure + * + * Return: 0 if there are waiters (wait_for_completion() in progress) + * 1 if there are no waiters. + * + */ +bool completion_done(struct completion *x) +{ + unsigned long flags; + int ret = 1; + + spin_lock_irqsave(&x->wait.lock, flags); + if (!x->done) + ret = 0; + spin_unlock_irqrestore(&x->wait.lock, flags); + return ret; +} +EXPORT_SYMBOL(completion_done); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 9dc2e236123b8..0c578d9e2c473 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -73,7 +73,7 @@ #include #include #include -#include +#include #include #include @@ -82,37 +82,17 @@ #ifdef CONFIG_PARAVIRT #include #endif +#include #include "sched.h" #include "../workqueue_internal.h" #include "../smpboot.h" - +#ifdef CONFIG_MTPROF +#include "mt_sched_mon.h" +#endif #define CREATE_TRACE_POINTS #include -const char *task_event_names[] = {"PUT_PREV_TASK", "PICK_NEXT_TASK", - "TASK_WAKE", "TASK_MIGRATE", "TASK_UPDATE", - "IRQ_UPDATE"}; - -//ATOMIC_NOTIFIER_HEAD(migration_notifier_head); -//ATOMIC_NOTIFIER_HEAD(load_alert_notifier_head); - -#ifdef smp_mb__before_atomic -void __smp_mb__before_atomic(void) -{ - smp_mb__before_atomic(); -} -EXPORT_SYMBOL(__smp_mb__before_atomic); -#endif - -#ifdef smp_mb__after_atomic -void __smp_mb__after_atomic(void) -{ - smp_mb__after_atomic(); -} -EXPORT_SYMBOL(__smp_mb__after_atomic); -#endif - void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period) { unsigned long delta; @@ -146,6 +126,8 @@ void update_rq_clock(struct rq *rq) return; delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; + if (delta < 0) + return; rq->clock += delta; update_rq_clock_task(rq, delta); } @@ -250,6 +232,7 @@ sched_feat_write(struct file *filp, const char __user *ubuf, char buf[64]; char *cmp; int i; + struct inode *inode; if (cnt > 63) cnt = 63; @@ -260,7 +243,11 @@ sched_feat_write(struct file *filp, const char __user *ubuf, buf[cnt] = 0; cmp = strstrip(buf); + /* Ensure the static_key remains in a consistent state */ + inode = file_inode(filp); + mutex_lock(&inode->i_mutex); i = sched_feat_set(cmp); + mutex_unlock(&inode->i_mutex); if (i == __SCHED_FEAT_NR) return -EINVAL; @@ -333,9 +320,12 @@ static inline struct rq *__task_rq_lock(struct task_struct *p) for (;;) { rq = task_rq(p); raw_spin_lock(&rq->lock); - if (likely(rq == task_rq(p))) + if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) return rq; raw_spin_unlock(&rq->lock); + + while (unlikely(task_on_rq_migrating(p))) + cpu_relax(); } } @@ -352,10 +342,13 @@ static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags) raw_spin_lock_irqsave(&p->pi_lock, *flags); rq = task_rq(p); raw_spin_lock(&rq->lock); - if (likely(rq == task_rq(p))) + if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) return rq; raw_spin_unlock(&rq->lock); raw_spin_unlock_irqrestore(&p->pi_lock, *flags); + + while (unlikely(task_on_rq_migrating(p))) + cpu_relax(); } } @@ -392,13 +385,6 @@ static struct rq *this_rq_lock(void) #ifdef CONFIG_SCHED_HRTICK /* * Use HR-timers to deliver accurate preemption points. - * - * Its all a bit involved since we cannot program an hrt while holding the - * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a - * reschedule event. - * - * When we get rescheduled we reprogram the hrtick_timer outside of the - * rq->lock. */ static void hrtick_clear(struct rq *rq) @@ -426,22 +412,24 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer) } #ifdef CONFIG_SMP + +static int __hrtick_restart(struct rq *rq) +{ + struct hrtimer *timer = &rq->hrtick_timer; + ktime_t time = hrtimer_get_softexpires(timer); + + return __hrtimer_start_range_ns(timer, time, 0, HRTIMER_MODE_ABS_PINNED, 0); +} + /* * called from hardirq (IPI) context */ static void __hrtick_start(void *arg) { struct rq *rq = arg; - struct hrtimer *timer = &rq->hrtick_timer; - ktime_t soft, hard; - unsigned long delta; - - soft = hrtimer_get_softexpires(timer); - hard = hrtimer_get_expires(timer); - delta = ktime_to_ns(ktime_sub(hard, soft)); raw_spin_lock(&rq->lock); - __hrtimer_start_range_ns(timer, soft, delta, HRTIMER_MODE_ABS, 0); + __hrtick_restart(rq); rq->hrtick_csd_pending = 0; raw_spin_unlock(&rq->lock); } @@ -454,15 +442,22 @@ static void __hrtick_start(void *arg) void hrtick_start(struct rq *rq, u64 delay) { struct hrtimer *timer = &rq->hrtick_timer; - ktime_t time = ktime_add_ns(timer->base->get_time(), delay); + ktime_t time; + s64 delta; + + /* + * Don't schedule slices shorter than 10000ns, that just + * doesn't make sense and can cause timer DoS. + */ + delta = max_t(s64, delay, 10000LL); + time = ktime_add_ns(timer->base->get_time(), delta); hrtimer_set_expires(timer, time); if (rq == this_rq()) { - __hrtimer_start_range_ns(timer, ns_to_ktime(delay), 0, - HRTIMER_MODE_REL_PINNED, 0); + __hrtick_restart(rq); } else if (!rq->hrtick_csd_pending) { - __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0); + smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); rq->hrtick_csd_pending = 1; } } @@ -498,6 +493,11 @@ static __init void init_hrtick(void) */ void hrtick_start(struct rq *rq, u64 delay) { + /* + * Don't schedule slices shorter than 10000ns, that just + * doesn't make sense. Rely on vruntime for fairness. + */ + delay = max_t(u64, delay, 10000LL); __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0, HRTIMER_MODE_REL_PINNED, 0); } @@ -535,32 +535,102 @@ static inline void init_hrtick(void) #endif /* CONFIG_SCHED_HRTICK */ /* - * resched_task - mark a task 'to be rescheduled now'. + * cmpxchg based fetch_or, macro so it works for different integer types + */ +#define fetch_or(ptr, val) \ +({ typeof(*(ptr)) __old, __val = *(ptr); \ + for (;;) { \ + __old = cmpxchg((ptr), __val, __val | (val)); \ + if (__old == __val) \ + break; \ + __val = __old; \ + } \ + __old; \ +}) + +#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG) +/* + * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, + * this avoids any races wrt polling state changes and thereby avoids + * spurious IPIs. + */ +static bool set_nr_and_not_polling(struct task_struct *p) +{ + struct thread_info *ti = task_thread_info(p); + return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); +} + +/* + * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. + * + * If this returns true, then the idle task promises to call + * sched_ttwu_pending() and reschedule soon. + */ +static bool set_nr_if_polling(struct task_struct *p) +{ + struct thread_info *ti = task_thread_info(p); + typeof(ti->flags) old, val = ACCESS_ONCE(ti->flags); + + for (;;) { + if (!(val & _TIF_POLLING_NRFLAG)) + return false; + if (val & _TIF_NEED_RESCHED) + return true; + old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED); + if (old == val) + break; + val = old; + } + return true; +} + +#else +static bool set_nr_and_not_polling(struct task_struct *p) +{ + set_tsk_need_resched(p); + return true; +} + +#ifdef CONFIG_SMP +static bool set_nr_if_polling(struct task_struct *p) +{ + return false; +} +#endif +#endif + +/* + * resched_curr - mark rq's current task 'to be rescheduled now'. * * On UP this means the setting of the need_resched flag, on SMP it * might also involve a cross-CPU call to trigger the scheduler on * the target CPU. */ -#ifdef CONFIG_SMP -void resched_task(struct task_struct *p) +void resched_curr(struct rq *rq) { + struct task_struct *curr = rq->curr; int cpu; - assert_raw_spin_locked(&task_rq(p)->lock); + lockdep_assert_held(&rq->lock); - if (test_tsk_need_resched(p)) + if (test_tsk_need_resched(curr)) return; - set_tsk_need_resched(p); + cpu = cpu_of(rq); - cpu = task_cpu(p); - if (cpu == smp_processor_id()) + if (cpu == smp_processor_id()) { + set_tsk_need_resched(curr); + set_preempt_need_resched(); return; + } - /* NEED_RESCHED must be visible before we test polling */ - smp_mb(); - if (!tsk_is_polling(p)) + if (set_nr_and_not_polling(curr)) smp_send_reschedule(cpu); + // TJK fix tracing +#if 0 + else + trace_sched_wake_idle_without_ipi(cpu); +#endif } void resched_cpu(int cpu) @@ -570,10 +640,11 @@ void resched_cpu(int cpu) if (!raw_spin_trylock_irqsave(&rq->lock, flags)) return; - resched_task(cpu_curr(cpu)); + resched_curr(rq); raw_spin_unlock_irqrestore(&rq->lock, flags); } +#ifdef CONFIG_SMP #ifdef CONFIG_NO_HZ_COMMON /* * In the semi idle case, use the nearest busy cpu for migrating timers @@ -619,31 +690,23 @@ static void wake_up_idle_cpu(int cpu) if (cpu == smp_processor_id()) return; - /* - * This is safe, as this function is called with the timer - * wheel base lock of (cpu) held. When the CPU is on the way - * to idle and has not yet set rq->curr to idle then it will - * be serialized on the timer wheel base lock and take the new - * timer into account automatically. - */ - if (rq->curr != rq->idle) - return; - - /* - * We can set TIF_RESCHED on the idle task of the other CPU - * lockless. The worst case is that the other CPU runs the - * idle task through an additional NOOP schedule() - */ - set_tsk_need_resched(rq->idle); - - /* NEED_RESCHED must be visible before we test polling */ - smp_mb(); - if (!tsk_is_polling(rq->idle)) + if (set_nr_and_not_polling(rq->idle)) smp_send_reschedule(cpu); +#if 0 + // TJK fix tracing + else + trace_sched_wake_idle_without_ipi(cpu); +#endif } static bool wake_up_full_nohz_cpu(int cpu) { + /* + * We just need the target to call irq_exit() and re-evaluate + * the next tick. The nohz full kick at least implies that. + * If needed we can still optimize that later with an + * empty IRQ. + */ if (tick_nohz_full_cpu(cpu)) { if (cpu != smp_processor_id() || tick_nohz_tick_stopped()) @@ -690,18 +753,15 @@ static inline bool got_nohz_idle_kick(void) #ifdef CONFIG_NO_HZ_FULL bool sched_can_stop_tick(void) { - struct rq *rq; - - rq = this_rq(); - - /* Make sure rq->nr_running update is visible after the IPI */ - smp_rmb(); - - /* More than one running task need preemption */ - if (rq->nr_running > 1) - return false; + /* + * More than one running task need preemption. + * nr_running update is assumed to be visible + * after IPI is sent from wakers. + */ + if (this_rq()->nr_running > 1) + return false; - return true; + return true; } #endif /* CONFIG_NO_HZ_FULL */ @@ -709,7 +769,7 @@ void sched_avg_update(struct rq *rq) { s64 period = sched_avg_period(); - while ((s64)(rq->clock - rq->age_stamp) > period) { + while ((s64)(rq_clock(rq) - rq->age_stamp) > period) { /* * Inline assembly required to prevent the compiler * optimising this loop into a divmod call. @@ -721,30 +781,6 @@ void sched_avg_update(struct rq *rq) } } -/* - * Note C-state for (idle) cpus. - * - * @cstate = cstate index, 0 -> active state - * @wakeup_energy = energy spent in waking up cpu - * @wakeup_latency = latency to wakeup from cstate - * - */ -void -sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency) -{ - struct rq *rq = cpu_rq(cpu); - - rq->cstate = cstate; /* C1, C2 etc */ - rq->wakeup_energy = wakeup_energy; - rq->wakeup_latency = wakeup_latency; -} - -#else /* !CONFIG_SMP */ -void resched_task(struct task_struct *p) -{ - assert_raw_spin_locked(&task_rq(p)->lock); - set_tsk_need_resched(p); -} #endif /* CONFIG_SMP */ #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ @@ -813,17 +849,15 @@ static void set_load_weight(struct task_struct *p) static void enqueue_task(struct rq *rq, struct task_struct *p, int flags) { update_rq_clock(rq); - sched_info_queued(p); + sched_info_queued(rq, p); p->sched_class->enqueue_task(rq, p, flags); - trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_allowed)[0]); } static void dequeue_task(struct rq *rq, struct task_struct *p, int flags) { update_rq_clock(rq); - sched_info_dequeued(p); + sched_info_dequeued(rq, p); p->sched_class->dequeue_task(rq, p, flags); - trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_allowed)[0]); } void activate_task(struct rq *rq, struct task_struct *p, int flags) @@ -877,19 +911,13 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) #endif #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING if (static_key_false((¶virt_steal_rq_enabled))) { - u64 st; - steal = paravirt_steal_clock(cpu_of(rq)); steal -= rq->prev_steal_time_rq; if (unlikely(steal > delta)) steal = delta; - st = steal_ticks(steal); - steal = st * TICK_NSEC; - rq->prev_steal_time_rq += steal; - delta -= steal; } #endif @@ -897,7 +925,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) rq->clock_task += delta; #if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING) - if ((irq_delta + steal) && sched_feat(NONTASK_POWER)) + if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) sched_rt_avg_update(rq, irq_delta + steal); #endif } @@ -951,7 +979,9 @@ static inline int normal_prio(struct task_struct *p) { int prio; - if (task_has_rt_policy(p)) + if (task_has_dl_policy(p)) + prio = MAX_DL_PRIO-1; + else if (task_has_rt_policy(p)) prio = MAX_RT_PRIO-1 - p->rt_priority; else prio = __normal_prio(p); @@ -981,6 +1011,8 @@ static int effective_prio(struct task_struct *p) /** * task_curr - is this task currently executing on a CPU? * @p: the task in question. + * + * Return: 1 if the task is currently executing. 0 otherwise. */ inline int task_curr(const struct task_struct *p) { @@ -995,7 +1027,7 @@ static inline void check_class_changed(struct rq *rq, struct task_struct *p, if (prev_class->switched_from) prev_class->switched_from(rq, p); p->sched_class->switched_to(rq, p); - } else if (oldprio != p->prio) + } else if (oldprio != p->prio || dl_task(p)) p->sched_class->prio_changed(rq, p, oldprio); } @@ -1010,7 +1042,7 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) if (class == rq->curr->sched_class) break; if (class == p->sched_class) { - resched_task(rq->curr); + resched_curr(rq); break; } } @@ -1020,4283 +1052,2069 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) * A queue event has occurred, and we're going to schedule. In * this case, we can save a useless back to back clock update. */ - if (rq->curr->on_rq && test_tsk_need_resched(rq->curr)) + if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) rq->skip_clock_update = 1; } -static ATOMIC_NOTIFIER_HEAD(task_migration_notifier); - -void register_task_migration_notifier(struct notifier_block *n) +#ifdef CONFIG_SMP +void set_task_cpu(struct task_struct *p, unsigned int new_cpu) { - atomic_notifier_chain_register(&task_migration_notifier, n); -} - -#ifdef CONFIG_SCHED_HMP +#ifdef CONFIG_SCHED_DEBUG + /* + * We should never call set_task_cpu() on a blocked task, + * ttwu() will sort out the placement. + */ + WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING && + !(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE)); -static int __init set_sched_enable_hmp(char *str) -{ - int enable_hmp = 0; +#ifdef CONFIG_LOCKDEP + /* + * The caller should hold either p->pi_lock or rq->lock, when changing + * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. + * + * sched_move_task() holds both and thus holding either pins the cgroup, + * see task_group(). + * + * Furthermore, all task_rq users should acquire both locks, see + * task_rq_lock(). + */ + WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || + lockdep_is_held(&task_rq(p)->lock))); +#endif +#endif - get_option(&str, &enable_hmp); + // TJK: convert to new trace args + trace_sched_migrate_task(p, new_cpu, 0); - sched_enable_hmp = !!enable_hmp; + if (task_cpu(p) != new_cpu) { + if (p->sched_class->migrate_task_rq) + p->sched_class->migrate_task_rq(p, new_cpu); + p->se.nr_migrations++; + perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0); + } - return 0; + __set_task_cpu(p, new_cpu); } -early_param("sched_enable_hmp", set_sched_enable_hmp); - -static int __init set_sched_enable_power_aware(char *str) +#ifdef TJK_HMP +static void __migrate_swap_task(struct task_struct *p, int cpu) { - int enable_power_aware = 0; - - get_option(&str, &enable_power_aware); + if (task_on_rq_queued(p)) { + struct rq *src_rq, *dst_rq; - sysctl_sched_enable_power_aware = !!enable_power_aware; + src_rq = task_rq(p); + dst_rq = cpu_rq(cpu); - return 0; + deactivate_task(src_rq, p, 0); + set_task_cpu(p, cpu); + activate_task(dst_rq, p, 0); + check_preempt_curr(dst_rq, p, 0); + } else { + /* + * Task isn't running anymore; make it appear like we migrated + * it before it went to sleep. This means on wakeup we make the + * previous cpu our targer instead of where it really is. + */ + p->wake_cpu = cpu; + } } +#endif -early_param("sched_enable_power_aware", set_sched_enable_power_aware); +struct migration_swap_arg { + struct task_struct *src_task, *dst_task; + int src_cpu, dst_cpu; +}; -static inline int got_boost_kick(void) +#ifdef TJK_HMP +static int migrate_swap_stop(void *data) { - int cpu = smp_processor_id(); - struct rq *rq = cpu_rq(cpu); + struct migration_swap_arg *arg = data; + struct rq *src_rq, *dst_rq; + int ret = -EAGAIN; - return test_bit(BOOST_KICK, &rq->hmp_flags); -} + src_rq = cpu_rq(arg->src_cpu); + dst_rq = cpu_rq(arg->dst_cpu); -static inline void clear_boost_kick(int cpu) -{ - struct rq *rq = cpu_rq(cpu); + double_raw_lock(&arg->src_task->pi_lock, + &arg->dst_task->pi_lock); + double_rq_lock(src_rq, dst_rq); + if (task_cpu(arg->dst_task) != arg->dst_cpu) + goto unlock; - clear_bit(BOOST_KICK, &rq->hmp_flags); -} + if (task_cpu(arg->src_task) != arg->src_cpu) + goto unlock; -void boost_kick(int cpu) -{ - struct rq *rq = cpu_rq(cpu); + if (!cpumask_test_cpu(arg->dst_cpu, tsk_cpus_allowed(arg->src_task))) + goto unlock; - if (!test_and_set_bit(BOOST_KICK, &rq->hmp_flags)) - smp_send_reschedule(cpu); -} + if (!cpumask_test_cpu(arg->src_cpu, tsk_cpus_allowed(arg->dst_task))) + goto unlock; -/* Clear any HMP scheduler related requests pending from or on cpu */ -static inline void clear_hmp_request(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - unsigned long flags; + __migrate_swap_task(arg->src_task, arg->dst_cpu); + __migrate_swap_task(arg->dst_task, arg->src_cpu); - clear_boost_kick(cpu); - clear_reserved(cpu); - if (rq->push_task) { - raw_spin_lock_irqsave(&rq->lock, flags); - if (rq->push_task) { - clear_reserved(rq->push_cpu); - put_task_struct(rq->push_task); - rq->push_task = NULL; - } - rq->active_balance = 0; - raw_spin_unlock_irqrestore(&rq->lock, flags); - } -} + ret = 0; -#else +unlock: + double_rq_unlock(src_rq, dst_rq); + raw_spin_unlock(&arg->dst_task->pi_lock); + raw_spin_unlock(&arg->src_task->pi_lock); -static inline int got_boost_kick(void) -{ - return 0; + return ret; } - -static inline void clear_boost_kick(int cpu) { } - -static inline void clear_hmp_request(int cpu) { } - -#endif /* CONFIG_SCHED_HMP */ - -#if defined(CONFIG_SCHED_HMP) +#endif /* - * sched_window_stats_policy, sched_account_wait_time, sched_ravg_hist_size, - * sched_migration_fixup, sched_freq_account_wait_time have a 'sysctl' copy - * associated with them. This is required for atomic update of those variables - * when being modifed via sysctl interface. - * - * IMPORTANT: Initialize both copies to same value!! + * Cross migrate two tasks */ +int migrate_swap(struct task_struct *cur, struct task_struct *p) +{ + struct migration_swap_arg arg; + int ret = -EINVAL; -static __read_mostly unsigned int sched_ravg_hist_size = 5; -__read_mostly unsigned int sysctl_sched_ravg_hist_size = 5; - -static __read_mostly unsigned int sched_window_stats_policy = - WINDOW_STATS_MAX_RECENT_AVG; -__read_mostly unsigned int sysctl_sched_window_stats_policy = - WINDOW_STATS_MAX_RECENT_AVG; - -static __read_mostly unsigned int sched_account_wait_time = 1; -__read_mostly unsigned int sysctl_sched_account_wait_time = 1; - -__read_mostly unsigned int sysctl_sched_cpu_high_irqload = (10 * NSEC_PER_MSEC); - -#ifdef CONFIG_SCHED_FREQ_INPUT + arg = (struct migration_swap_arg){ + .src_task = cur, + .src_cpu = task_cpu(cur), + .dst_task = p, + .dst_cpu = task_cpu(p), + }; -static __read_mostly unsigned int sched_migration_fixup = 1; -__read_mostly unsigned int sysctl_sched_migration_fixup = 1; + if (arg.src_cpu == arg.dst_cpu) + goto out; -static __read_mostly unsigned int sched_freq_account_wait_time; -__read_mostly unsigned int sysctl_sched_freq_account_wait_time; + /* + * These three tests are all lockless; this is OK since all of them + * will be re-checked with proper locks held further down the line. + */ + if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu)) + goto out; -/* - * For increase, send notification if - * freq_required - cur_freq > sysctl_sched_freq_inc_notify - */ -__read_mostly int sysctl_sched_freq_inc_notify = 10 * 1024 * 1024; /* + 10GHz */ + if (!cpumask_test_cpu(arg.dst_cpu, tsk_cpus_allowed(arg.src_task))) + goto out; -/* - * For decrease, send notification if - * cur_freq - freq_required > sysctl_sched_freq_dec_notify - */ -__read_mostly int sysctl_sched_freq_dec_notify = 10 * 1024 * 1024; /* - 10GHz */ + if (!cpumask_test_cpu(arg.src_cpu, tsk_cpus_allowed(arg.dst_task))) + goto out; -static __read_mostly unsigned int sched_io_is_busy; +#if 0 + trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); + ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); +#else + ret = -EINVAL; +#endif -#endif /* CONFIG_SCHED_FREQ_INPUT */ +out: + return ret; +} -/* 1 -> use PELT based load stats, 0 -> use window-based load stats */ -unsigned int __read_mostly sched_use_pelt; +struct migration_arg { + struct task_struct *task; + int dest_cpu; +}; -unsigned int max_possible_efficiency = 1024; -unsigned int min_possible_efficiency = 1024; +static int migration_cpu_stop(void *data); /* - * Maximum possible frequency across all cpus. Task demand and cpu - * capacity (cpu_power) metrics are scaled in reference to it. + * wait_task_inactive - wait for a thread to unschedule. + * + * If @match_state is nonzero, it's the @p->state value just checked and + * not expected to change. If it changes, i.e. @p might have woken up, + * then return zero. When we succeed in waiting for @p to be off its CPU, + * we return a positive number (its total switch count). If a second call + * a short while later returns the same number, the caller can be sure that + * @p has remained unscheduled the whole time. + * + * The caller must ensure that the task *will* unschedule sometime soon, + * else this function might spin for a *long* time. This function can't + * be called with interrupts off, or it may introduce deadlock with + * smp_call_function() if an IPI is sent by the same process we are + * waiting to become inactive. */ -unsigned int max_possible_freq = 1; +unsigned long wait_task_inactive(struct task_struct *p, long match_state) +{ + unsigned long flags; + int running, queued; + unsigned long ncsw; + struct rq *rq; -/* - * Minimum possible max_freq across all cpus. This will be same as - * max_possible_freq on homogeneous systems and could be different from - * max_possible_freq on heterogenous systems. min_max_freq is used to derive - * capacity (cpu_power) of cpus. - */ -unsigned int min_max_freq = 1; + for (;;) { + /* + * We do the initial early heuristics without holding + * any task-queue locks at all. We'll only try to get + * the runqueue lock when things look like they will + * work out! + */ + rq = task_rq(p); -unsigned int max_capacity = 1024; /* max(rq->capacity) */ -unsigned int min_capacity = 1024; /* min(rq->capacity) */ -unsigned int max_load_scale_factor = 1024; /* max possible load scale factor */ -unsigned int max_possible_capacity = 1024; /* max(rq->max_possible_capacity) */ - -/* Mask of all CPUs that have max_possible_capacity */ -cpumask_t mpc_mask = CPU_MASK_ALL; + /* + * If the task is actively running on another CPU + * still, just relax and busy-wait without holding + * any locks. + * + * NOTE! Since we don't hold any locks, it's not + * even sure that "rq" stays as the right runqueue! + * But we don't care, since "task_running()" will + * return false if the runqueue has changed and p + * is actually now running somewhere else! + */ + while (task_running(rq, p)) { + if (match_state && unlikely(p->state != match_state)) + return 0; + cpu_relax(); + } -/* Window size (in ns) */ -__read_mostly unsigned int sched_ravg_window = 10000000; + /* + * Ok, time to look more closely! We need the rq + * lock now, to be *sure*. If we're wrong, we'll + * just go back and repeat. + */ + rq = task_rq_lock(p, &flags); + trace_sched_wait_task(p); + running = task_running(rq, p); + queued = task_on_rq_queued(p); + ncsw = 0; + if (!match_state || p->state == match_state) + ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ + task_rq_unlock(rq, p, &flags); -/* Min window size (in ns) = 10ms */ -#define MIN_SCHED_RAVG_WINDOW 10000000 + /* + * If it changed from the expected state, bail out now. + */ + if (unlikely(!ncsw)) + break; -/* Max window size (in ns) = 1s */ -#define MAX_SCHED_RAVG_WINDOW 1000000000 + /* + * Was it really running after all now that we + * checked with the proper locks actually held? + * + * Oops. Go back and try again.. + */ + if (unlikely(running)) { + cpu_relax(); + continue; + } -/* Temporarily disable window-stats activity on all cpus */ -unsigned int __read_mostly sched_disable_window_stats; + /* + * It's not enough that it's not actively running, + * it must be off the runqueue _entirely_, and not + * preempted! + * + * So if it was still runnable (but just not actively + * running right now), it's preempted, and we should + * yield - it could be a while. + */ + if (unlikely(queued)) { + ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ); -static unsigned int sync_cpu; + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_hrtimeout(&to, HRTIMER_MODE_REL); + continue; + } -#define EXITING_TASK_MARKER 0xdeaddead + /* + * Ahh, all good. It wasn't running, and it wasn't + * runnable, which means that it will never become + * running in the future either. We're all done! + */ + break; + } -static inline int exiting_task(struct task_struct *p) -{ - return (p->ravg.sum_history[0] == EXITING_TASK_MARKER); + return ncsw; } -static int __init set_sched_ravg_window(char *str) +/*** + * kick_process - kick a running thread to enter/exit the kernel + * @p: the to-be-kicked thread + * + * Cause a process which is running on another CPU to enter + * kernel-mode, without any delay. (to get signals handled.) + * + * NOTE: this function doesn't have to take the runqueue lock, + * because all it wants to ensure is that the remote task enters + * the kernel. If the IPI races and the task has been migrated + * to another CPU then no harm is done and the purpose has been + * achieved as well. + */ +void kick_process(struct task_struct *p) { - get_option(&str, &sched_ravg_window); - - sched_use_pelt = (sched_ravg_window < MIN_SCHED_RAVG_WINDOW || - sched_ravg_window > MAX_SCHED_RAVG_WINDOW); + int cpu; - return 0; + preempt_disable(); + cpu = task_cpu(p); + if ((cpu != smp_processor_id()) && task_curr(p)) + smp_send_reschedule(cpu); + preempt_enable(); } +EXPORT_SYMBOL_GPL(kick_process); +#endif /* CONFIG_SMP */ -early_param("sched_ravg_window", set_sched_ravg_window); - -static inline void -update_window_start(struct rq *rq, u64 wallclock) +#ifdef CONFIG_SMP +/* + * ->cpus_allowed is protected by both rq->lock and p->pi_lock + */ +static int select_fallback_rq(int cpu, struct task_struct *p) { - s64 delta; - int nr_windows; + int nid = cpu_to_node(cpu); + const struct cpumask *nodemask = NULL; + enum { cpuset, possible, fail } state = cpuset; + int dest_cpu; - delta = wallclock - rq->window_start; - BUG_ON(delta < 0); - if (delta < sched_ravg_window) - return; + /* + * If the node that the cpu is on has been offlined, cpu_to_node() + * will return -1. There is no cpu on the node, and we should + * select the cpu on the other node. + */ + if (nid != -1) { + nodemask = cpumask_of_node(nid); - nr_windows = div64_u64(delta, sched_ravg_window); - rq->window_start += (u64)nr_windows * (u64)sched_ravg_window; -} + /* Look for allowed, online CPU in same node. */ + for_each_cpu(dest_cpu, nodemask) { + if (!cpu_online(dest_cpu)) + continue; + if (!cpu_active(dest_cpu)) + continue; + if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) + return dest_cpu; + } + } -static inline u64 scale_exec_time(u64 delta, struct rq *rq) -{ - unsigned int cur_freq = rq->cur_freq; - int sf; + for (;;) { + /* Any allowed, online CPU? */ + for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) { + if (!cpu_online(dest_cpu)) + continue; + if (!cpu_active(dest_cpu)) + continue; + goto out; + } - if (unlikely(cur_freq > max_possible_freq || - (cur_freq == rq->max_freq && - rq->max_freq < rq->max_possible_freq))) - cur_freq = rq->max_possible_freq; + switch (state) { + case cpuset: + /* No more Mr. Nice Guy. */ + cpuset_cpus_allowed_fallback(p); + state = possible; + break; - /* round up div64 */ - delta = div64_u64(delta * cur_freq + max_possible_freq - 1, - max_possible_freq); + case possible: + do_set_cpus_allowed(p, cpu_possible_mask); + state = fail; + break; - sf = DIV_ROUND_UP(rq->efficiency * 1024, max_possible_efficiency); + case fail: + BUG(); + break; + } + } - delta *= sf; - delta >>= 10; +out: + if (state != cpuset) { + /* + * Don't tell them about moving exiting tasks or + * kernel threads (both mm NULL), since they never + * leave kernel. + */ + if (p->mm && printk_ratelimit()) { + printk_deferred("process %d (%s) no longer affine to cpu%d\n", + task_pid_nr(p), p->comm, cpu); + } + } - return delta; + return dest_cpu; } -#ifdef CONFIG_SCHED_FREQ_INPUT - -static inline int cpu_is_waiting_on_io(struct rq *rq) +/* + * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable. + */ +static inline +int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) { - if (!sched_io_is_busy) - return 0; + cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags); + + /* + * In order not to call set_task_cpu() on a blocking task we need + * to rely on ttwu() to place the task on a valid ->cpus_allowed + * cpu. + * + * Since this is common to all placement strategies, this lives here. + * + * [ this allows ->select_task() to simply return task_cpu(p) and + * not worry about this generic constraint ] + */ + if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) || + !cpu_online(cpu))) + cpu = select_fallback_rq(task_cpu(p), p); - return atomic_read(&rq->nr_iowait); + return cpu; } -/* Does freq_required sufficiently exceed or fall behind cur_freq? */ -static inline int -nearly_same_freq(unsigned int cur_freq, unsigned int freq_required) +static void update_avg(u64 *avg, u64 sample) { - int delta = freq_required - cur_freq; - - if (freq_required > cur_freq) - return delta < sysctl_sched_freq_inc_notify; - - delta = -delta; - - return delta < sysctl_sched_freq_dec_notify; + s64 diff = sample - *avg; + *avg += diff >> 3; } +#endif -/* Convert busy time to frequency equivalent */ -static inline unsigned int load_to_freq(struct rq *rq, u64 load) +static void +ttwu_stat(struct task_struct *p, int cpu, int wake_flags) { - unsigned int freq; +#ifdef CONFIG_SCHEDSTATS + struct rq *rq = this_rq(); - load = scale_load_to_cpu(load, cpu_of(rq)); - load *= 128; - load = div64_u64(load, max_task_load()); +#ifdef CONFIG_SMP + int this_cpu = smp_processor_id(); - freq = load * rq->max_possible_freq; - freq /= 128; + if (cpu == this_cpu) { + schedstat_inc(rq, ttwu_local); + schedstat_inc(p, se.statistics.nr_wakeups_local); + } else { + struct sched_domain *sd; - return freq; -} + schedstat_inc(p, se.statistics.nr_wakeups_remote); + rcu_read_lock(); + for_each_domain(this_cpu, sd) { + if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { + schedstat_inc(sd, ttwu_wake_remote); + break; + } + } + rcu_read_unlock(); + } -/* Should scheduler alert governor for changing frequency? */ -static int send_notification(struct rq *rq) -{ - unsigned int cur_freq, freq_required; + if (wake_flags & WF_MIGRATED) + schedstat_inc(p, se.statistics.nr_wakeups_migrate); - if (!sched_enable_hmp) - return 0; +#endif /* CONFIG_SMP */ - cur_freq = load_to_freq(rq, rq->old_busy_time); - freq_required = load_to_freq(rq, rq->prev_runnable_sum); + schedstat_inc(rq, ttwu_count); + schedstat_inc(p, se.statistics.nr_wakeups); - if (nearly_same_freq(cur_freq, freq_required)) - return 0; + if (wake_flags & WF_SYNC) + schedstat_inc(p, se.statistics.nr_wakeups_sync); - return 1; +#endif /* CONFIG_SCHEDSTATS */ } -/* Alert governor if there is a need to change frequency */ -void check_for_freq_change(struct rq *rq) +static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags) { - int cpu = cpu_of(rq); - - if (!send_notification(rq)) - return; - - trace_sched_freq_alert(cpu, rq->old_busy_time, rq->prev_runnable_sum); + activate_task(rq, p, en_flags); + p->on_rq = TASK_ON_RQ_QUEUED; - atomic_notifier_call_chain( - &load_alert_notifier_head, 0, - (void *)(long)cpu); + /* if a worker is waking up, notify workqueue */ + if (p->flags & PF_WQ_WORKER) + wq_worker_waking_up(p, cpu_of(rq)); } -static int account_busy_for_cpu_time(struct rq *rq, struct task_struct *p, - u64 irqtime, int event) +/* + * Mark the task runnable and perform wakeup-preemption. + */ +static void +ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) { - if (is_idle_task(p)) { - /* TASK_WAKE && TASK_MIGRATE is not possible on idle task! */ - if (event == PICK_NEXT_TASK) - return 0; + check_preempt_curr(rq, p, wake_flags); + trace_sched_wakeup(p, true); - /* PUT_PREV_TASK, TASK_UPDATE && IRQ_UPDATE are left */ - return irqtime || cpu_is_waiting_on_io(rq); - } + p->state = TASK_RUNNING; +#ifdef CONFIG_SMP + if (p->sched_class->task_woken) + p->sched_class->task_woken(rq, p); - if (event == TASK_WAKE) - return 0; + if (rq->idle_stamp) { + u64 delta = rq_clock(rq) - rq->idle_stamp; + u64 max = 2*rq->max_idle_balance_cost; - if (event == PUT_PREV_TASK || event == IRQ_UPDATE || - event == TASK_UPDATE) - return 1; + update_avg(&rq->avg_idle, delta); + + if (rq->avg_idle > max) + rq->avg_idle = max; - /* Only TASK_MIGRATE && PICK_NEXT_TASK left */ - return sched_freq_account_wait_time; + rq->idle_stamp = 0; + } +#endif } -static inline int -heavy_task_wakeup(struct task_struct *p, struct rq *rq, int event) +static void +ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags) { - u32 task_demand = p->ravg.demand; +#ifdef CONFIG_SMP + if (p->sched_contributes_to_load) + rq->nr_uninterruptible--; +#endif - if (!sched_heavy_task || event != TASK_WAKE || - task_demand < sched_heavy_task || exiting_task(p)) - return 0; - - if (p->ravg.mark_start > rq->window_start) - return 0; - - /* has a full window elapsed since task slept? */ - return (rq->window_start - p->ravg.mark_start > sched_ravg_window); + ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING); + ttwu_do_wakeup(rq, p, wake_flags); } /* - * Account cpu activity in its busy time counters (rq->curr/prev_runnable_sum) + * Called in case the task @p isn't fully descheduled from its runqueue, + * in this case we must do a remote wakeup. Its a 'light' wakeup though, + * since all we need to do is flip p->state to TASK_RUNNING, since + * the task is still ->on_rq. */ -static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, - int event, u64 wallclock, u64 irqtime) +static int ttwu_remote(struct task_struct *p, int wake_flags) { - int new_window, nr_full_windows = 0; - int p_is_curr_task = (p == rq->curr); - u64 mark_start = p->ravg.mark_start; - u64 window_start = rq->window_start; - u32 window_size = sched_ravg_window; - u64 delta; - - new_window = mark_start < window_start; - if (new_window) - nr_full_windows = div64_u64((window_start - mark_start), - window_size); - - /* Handle per-task window rollover. We don't care about the idle - * task or exiting tasks. */ - if (new_window && !is_idle_task(p) && !exiting_task(p)) { - u32 curr_window = 0; - - if (!nr_full_windows) - curr_window = p->ravg.curr_window; + struct rq *rq; + int ret = 0; - p->ravg.prev_window = curr_window; - p->ravg.curr_window = 0; + rq = __task_rq_lock(p); + if (task_on_rq_queued(p)) { + /* check_preempt_curr() may use rq clock */ + update_rq_clock(rq); + ttwu_do_wakeup(rq, p, wake_flags); + ret = 1; } + __task_rq_unlock(rq); - if (!account_busy_for_cpu_time(rq, p, irqtime, event)) { - /* account_busy_for_cpu_time() = 0, so no update to the - * task's current window needs to be made. This could be - * for example - * - * - a wakeup event on a task within the current - * window (!new_window below, no action required), - * - switching to a new task from idle (PICK_NEXT_TASK) - * in a new window where irqtime is 0 and we aren't - * waiting on IO */ - - if (!new_window) - return; + return ret; +} - /* A new window has started. The RQ demand must be rolled - * over if p is the current task. */ - if (p_is_curr_task) { - u64 prev_sum = 0; - - /* p is either idle task or an exiting task */ - if (!nr_full_windows) - prev_sum = rq->curr_runnable_sum; - - rq->prev_runnable_sum = prev_sum; - rq->curr_runnable_sum = 0; - - } else if (heavy_task_wakeup(p, rq, event)) { - /* A new window has started. If p is a waking - * heavy task its prev_window contribution is faked - * to be its window-based demand. Note that this can - * introduce phantom load into the system depending - * on the window policy and task behavior. This feature - * can be controlled via the sched_heavy_task - * tunable. */ - p->ravg.prev_window = p->ravg.demand; - rq->prev_runnable_sum += p->ravg.demand; - } +#ifdef CONFIG_SMP +void sched_ttwu_pending(void) +{ + struct rq *rq = this_rq(); + struct llist_node *llist = llist_del_all(&rq->wake_list); + struct task_struct *p; + unsigned long flags; + if (!llist) return; - } - if (!new_window) { - /* account_busy_for_cpu_time() = 1 so busy time needs - * to be accounted to the current window. No rollover - * since we didn't start a new window. An example of this is - * when a task starts execution and then sleeps within the - * same window. */ - - if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq)) - delta = wallclock - mark_start; - else - delta = irqtime; - delta = scale_exec_time(delta, rq); - rq->curr_runnable_sum += delta; - if (!is_idle_task(p) && !exiting_task(p)) - p->ravg.curr_window += delta; + raw_spin_lock_irqsave(&rq->lock, flags); - return; + while (llist) { + p = llist_entry(llist, struct task_struct, wake_entry); + llist = llist_next(llist); + ttwu_do_activate(rq, p, 0); } - if (!p_is_curr_task) { - /* account_busy_for_cpu_time() = 1 so busy time needs - * to be accounted to the current window. A new window - * has also started, but p is not the current task, so the - * window is not rolled over - just split up and account - * as necessary into curr and prev. The window is only - * rolled over when a new window is processed for the current - * task. - * - * Irqtime can't be accounted by a task that isn't the - * currently running task. */ - - if (!nr_full_windows) { - /* A full window hasn't elapsed, account partial - * contribution to previous completed window. */ - delta = scale_exec_time(window_start - mark_start, rq); - if (!exiting_task(p)) - p->ravg.prev_window += delta; - } else { - /* Since at least one full window has elapsed, - * the contribution to the previous window is the - * full window (window_size). */ - delta = scale_exec_time(window_size, rq); - if (!exiting_task(p)) - p->ravg.prev_window = delta; - } - rq->prev_runnable_sum += delta; - - /* Account piece of busy time in the current window. */ - delta = scale_exec_time(wallclock - window_start, rq); - rq->curr_runnable_sum += delta; - if (!exiting_task(p)) - p->ravg.curr_window = delta; - - return; - } + raw_spin_unlock_irqrestore(&rq->lock, flags); +} - if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq)) { - /* account_busy_for_cpu_time() = 1 so busy time needs - * to be accounted to the current window. A new window - * has started and p is the current task so rollover is - * needed. If any of these three above conditions are true - * then this busy time can't be accounted as irqtime. - * - * Busy time for the idle task or exiting tasks need not - * be accounted. - * - * An example of this would be a task that starts execution - * and then sleeps once a new window has begun. */ - - if (!nr_full_windows) { - /* A full window hasn't elapsed, account partial - * contribution to previous completed window. */ - delta = scale_exec_time(window_start - mark_start, rq); - if (!is_idle_task(p) && !exiting_task(p)) - p->ravg.prev_window += delta; - delta += rq->curr_runnable_sum; - } else { - /* Since at least one full window has elapsed, - * the contribution to the previous window is the - * full window (window_size). */ - delta = scale_exec_time(window_size, rq); - if (!is_idle_task(p) && !exiting_task(p)) - p->ravg.prev_window = delta; - } - /* Rollover is done here by overwriting the values in - * prev_runnable_sum and curr_runnable_sum. */ - rq->prev_runnable_sum = delta; +enum ipi_msg_type { + IPI_RESCHEDULE, + IPI_CALL_FUNC, + IPI_CALL_FUNC_SINGLE, + IPI_CPU_STOP, + IPI_TIMER, + IPI_IRQ_WORK, +}; - /* Account piece of busy time in the current window. */ - delta = scale_exec_time(wallclock - window_start, rq); - rq->curr_runnable_sum = delta; - if (!is_idle_task(p) && !exiting_task(p)) - p->ravg.curr_window = delta; +void scheduler_ipi(void) +{ + /* + * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting + * TIF_NEED_RESCHED remotely (for the first time) will also send + * this IPI. + */ + preempt_fold_need_resched(); + if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick()) { +#ifdef CONFIG_MTPROF + mt_trace_ISR_start(IPI_RESCHEDULE); + mt_trace_ISR_end(IPI_RESCHEDULE); +#endif return; } - if (irqtime) { - /* account_busy_for_cpu_time() = 1 so busy time needs - * to be accounted to the current window. A new window - * has started and p is the current task so rollover is - * needed. The current task must be the idle task because - * irqtime is not accounted for any other task. - * - * Irqtime will be accounted each time we process IRQ activity - * after a period of idleness, so we know the IRQ busy time - * started at wallclock - irqtime. */ - - BUG_ON(!is_idle_task(p)); - mark_start = wallclock - irqtime; - - /* Roll window over. If IRQ busy time was just in the current - * window then that is all that need be accounted. */ - rq->prev_runnable_sum = rq->curr_runnable_sum; - if (mark_start > window_start) { - rq->curr_runnable_sum = scale_exec_time(irqtime, rq); - return; - } - - /* The IRQ busy time spanned multiple windows. Process the - * busy time preceding the current window start first. */ - delta = window_start - mark_start; - if (delta > window_size) - delta = window_size; - delta = scale_exec_time(delta, rq); - rq->prev_runnable_sum += delta; - - /* Process the remaining IRQ busy time in the current window. */ - delta = wallclock - window_start; - rq->curr_runnable_sum = scale_exec_time(delta, rq); + /* + * Not all reschedule IPI handlers call irq_enter/irq_exit, since + * traditionally all their work was done from the interrupt return + * path. Now that we actually do some work, we need to make sure + * we do call them. + * + * Some archs already do call them, luckily irq_enter/exit nest + * properly. + * + * Arguably we should visit all archs and update all handlers, + * however a fair share of IPIs are still resched only so this would + * somewhat pessimize the simple resched case. + */ + irq_enter(); +#ifdef CONFIG_MTPROF + mt_trace_ISR_start(IPI_RESCHEDULE); +#endif + sched_ttwu_pending(); - return; + /* + * Check if someone kicked us for doing the nohz idle load balance. + */ + if (unlikely(got_nohz_idle_kick())) { + this_rq()->idle_balance = 1; + raise_softirq_irqoff(SCHED_SOFTIRQ); } - - BUG(); +#ifdef CONFIG_MTPROF + mt_trace_ISR_end(IPI_RESCHEDULE); +#endif + irq_exit(); } -#else /* CONFIG_SCHED_FREQ_INPUT */ -static inline void update_cpu_busy_time(struct task_struct *p, struct rq *rq, - int event, u64 wallclock, u64 irqtime) +static void ttwu_queue_remote(struct task_struct *p, int cpu) { -} + struct rq *rq = cpu_rq(cpu); -#endif /* CONFIG_SCHED_FREQ_INPUT */ + if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) { + if (!set_nr_if_polling(rq->idle)) + smp_send_reschedule(cpu); +#if 0 + // TJK: fix tracing + else + trace_sched_wake_idle_without_ipi(cpu); +#endif + } +} -static int account_busy_for_task_demand(struct task_struct *p, int event) +void wake_up_if_idle(int cpu) { - /* No need to bother updating task demand for exiting tasks - * or the idle task. */ - if (exiting_task(p) || is_idle_task(p)) - return 0; - - /* When a task is waking up it is completing a segment of non-busy - * time. Likewise, if wait time is not treated as busy time, then - * when a task begins to run or is migrated, it is not running and - * is completing a segment of non-busy time. */ - if (event == TASK_WAKE || (!sched_account_wait_time && - (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) - return 0; - - return 1; -} + struct rq *rq = cpu_rq(cpu); + unsigned long flags; -/* - * Called when new window is starting for a task, to record cpu usage over - * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 - * when, say, a real-time task runs without preemption for several windows at a - * stretch. - */ -static void update_history(struct rq *rq, struct task_struct *p, - u32 runtime, int samples, int event) -{ - u32 *hist = &p->ravg.sum_history[0]; - int ridx, widx; - u32 max = 0, avg, demand; - u64 sum = 0; - - /* Ignore windows where task had no activity */ - if (!runtime || is_idle_task(p) || exiting_task(p) || !samples) - goto done; - - /* Push new 'runtime' value onto stack */ - widx = sched_ravg_hist_size - 1; - ridx = widx - samples; - for (; ridx >= 0; --widx, --ridx) { - hist[widx] = hist[ridx]; - sum += hist[widx]; - if (hist[widx] > max) - max = hist[widx]; - } - - for (widx = 0; widx < samples && widx < sched_ravg_hist_size; widx++) { - hist[widx] = runtime; - sum += hist[widx]; - if (hist[widx] > max) - max = hist[widx]; - } - - p->ravg.sum = 0; - if (p->on_rq) - p->sched_class->dec_hmp_sched_stats(rq, p); - - avg = div64_u64(sum, sched_ravg_hist_size); - - if (sched_window_stats_policy == WINDOW_STATS_RECENT) - demand = runtime; - else if (sched_window_stats_policy == WINDOW_STATS_MAX) - demand = max; - else if (sched_window_stats_policy == WINDOW_STATS_AVG) - demand = avg; - else - demand = max(avg, runtime); + rcu_read_lock(); - p->ravg.demand = demand; + if (!is_idle_task(rcu_dereference(rq->curr))) + goto out; - if (p->on_rq) - p->sched_class->inc_hmp_sched_stats(rq, p); + if (set_nr_if_polling(rq->idle)) { + // TJK: trace_sched_wake_idle_without_ipi(cpu); + } else { + raw_spin_lock_irqsave(&rq->lock, flags); + if (is_idle_task(rq->curr)) + smp_send_reschedule(cpu); + /* Else cpu is not in idle, do nothing here */ + raw_spin_unlock_irqrestore(&rq->lock, flags); + } -done: - trace_sched_update_history(rq, p, runtime, samples, event); +out: + rcu_read_unlock(); } -static void add_to_task_demand(struct rq *rq, struct task_struct *p, - u64 delta) +bool cpus_share_cache(int this_cpu, int that_cpu) { - delta = scale_exec_time(delta, rq); - p->ravg.sum += delta; - if (unlikely(p->ravg.sum > sched_ravg_window)) - p->ravg.sum = sched_ravg_window; + return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); } +#endif /* CONFIG_SMP */ -/* - * Account cpu demand of task and/or update task's cpu demand history - * - * ms = p->ravg.mark_start; - * wc = wallclock - * ws = rq->window_start - * - * Three possibilities: - * - * a) Task event is contained within one window. - * window_start < mark_start < wallclock - * - * ws ms wc - * | | | - * V V V - * |---------------| - * - * In this case, p->ravg.sum is updated *iff* event is appropriate - * (ex: event == PUT_PREV_TASK) - * - * b) Task event spans two windows. - * mark_start < window_start < wallclock - * - * ms ws wc - * | | | - * V V V - * -----|------------------- - * - * In this case, p->ravg.sum is updated with (ws - ms) *iff* event - * is appropriate, then a new window sample is recorded followed - * by p->ravg.sum being set to (wc - ws) *iff* event is appropriate. - * - * c) Task event spans more than two windows. - * - * ms ws_tmp ws wc - * | | | | - * V V V V - * ---|-------|-------|-------|-------|------ - * | | - * |<------ nr_full_windows ------>| - * - * In this case, p->ravg.sum is updated with (ws_tmp - ms) first *iff* - * event is appropriate, window sample of p->ravg.sum is recorded, - * 'nr_full_window' samples of window_size is also recorded *iff* - * event is appropriate and finally p->ravg.sum is set to (wc - ws) - * *iff* event is appropriate. - * - * IMPORTANT : Leave p->ravg.mark_start unchanged, as update_cpu_busy_time() - * depends on it! - */ -static void update_task_demand(struct task_struct *p, struct rq *rq, - int event, u64 wallclock) -{ - u64 mark_start = p->ravg.mark_start; - u64 delta, window_start = rq->window_start; - int new_window, nr_full_windows; - u32 window_size = sched_ravg_window; - - new_window = mark_start < window_start; - if (!account_busy_for_task_demand(p, event)) { - if (new_window) - /* If the time accounted isn't being accounted as - * busy time, and a new window started, only the - * previous window need be closed out with the - * pre-existing demand. Multiple windows may have - * elapsed, but since empty windows are dropped, - * it is not necessary to account those. */ - update_history(rq, p, p->ravg.sum, 1, event); - return; - } +static void ttwu_queue(struct task_struct *p, int cpu) +{ + struct rq *rq = cpu_rq(cpu); - if (!new_window) { - /* The simple case - busy time contained within the existing - * window. */ - add_to_task_demand(rq, p, wallclock - mark_start); +#if defined(CONFIG_SMP) + if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { + sched_clock_cpu(cpu); /* sync clocks x-cpu */ + ttwu_queue_remote(p, cpu); return; } +#endif - /* Busy time spans at least two windows. Temporarily rewind - * window_start to first window boundary after mark_start. */ - delta = window_start - mark_start; - nr_full_windows = div64_u64(delta, window_size); - window_start -= (u64)nr_full_windows * (u64)window_size; - - /* Process (window_start - mark_start) first */ - add_to_task_demand(rq, p, window_start - mark_start); - - /* Push new sample(s) into task's demand history */ - update_history(rq, p, p->ravg.sum, 1, event); - if (nr_full_windows) - update_history(rq, p, scale_exec_time(window_size, rq), - nr_full_windows, event); - - /* Roll window_start back to current to process any remainder - * in current window. */ - window_start += (u64)nr_full_windows * (u64)window_size; - - /* Process (wallclock - window_start) next */ - mark_start = window_start; - add_to_task_demand(rq, p, wallclock - mark_start); + raw_spin_lock(&rq->lock); + ttwu_do_activate(rq, p, 0); + raw_spin_unlock(&rq->lock); } -/* Reflect task activity on its demand and cpu's busy time statistics */ -static void update_task_ravg(struct task_struct *p, struct rq *rq, - int event, u64 wallclock, u64 irqtime) +/** + * try_to_wake_up - wake up a thread + * @p: the thread to be awakened + * @state: the mask of task states that can be woken + * @wake_flags: wake modifier flags (WF_*) + * + * Put it on the run-queue if it's not already there. The "current" + * thread is always on the run-queue (except when the actual + * re-schedule is in progress), and as such you're allowed to do + * the simpler "current->state = TASK_RUNNING" to mark yourself + * runnable without the overhead of this. + * + * Return: %true if @p was woken up, %false if it was already running. + * or @state didn't match @p's state. + */ +static int +try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) { - if (sched_use_pelt || !rq->window_start || sched_disable_window_stats) - return; - - lockdep_assert_held(&rq->lock); - - update_window_start(rq, wallclock); - - if (!p->ravg.mark_start) - goto done; - - update_task_demand(p, rq, event, wallclock); - update_cpu_busy_time(p, rq, event, wallclock, irqtime); - -done: - trace_sched_update_task_ravg(p, rq, event, wallclock, irqtime); + unsigned long flags; + int cpu, success = 0; - p->ravg.mark_start = wallclock; -} + /* + * If we are going to wake up a thread waiting for CONDITION we + * need to ensure that CONDITION=1 done by the caller can not be + * reordered with p->state check below. This pairs with mb() in + * set_current_state() the waiting thread does. + */ + smp_mb__before_spinlock(); + raw_spin_lock_irqsave(&p->pi_lock, flags); + if (!(p->state & state)) + goto out; -void sched_account_irqtime(int cpu, struct task_struct *curr, - u64 delta, u64 wallclock) -{ - struct rq *rq = cpu_rq(cpu); - unsigned long flags, nr_windows; - u64 cur_jiffies_ts, now; + success = 1; /* we're going to change ->state */ + cpu = task_cpu(p); - raw_spin_lock_irqsave(&rq->lock, flags); + if (p->on_rq && ttwu_remote(p, wake_flags)) + goto stat; - now = sched_clock(); - delta += (now - wallclock); - cur_jiffies_ts = get_jiffies_64(); +#ifdef CONFIG_SMP + /* + * If the owning (remote) cpu is still in the middle of schedule() with + * this task as prev, wait until its done referencing the task. + */ + while (p->on_cpu) + cpu_relax(); + /* + * Pairs with the smp_wmb() in finish_lock_switch(). + */ + smp_rmb(); - if (is_idle_task(curr)) - update_task_ravg(curr, rq, IRQ_UPDATE, now, delta); + p->sched_contributes_to_load = !!task_contributes_to_load(p); + p->state = TASK_WAKING; - nr_windows = cur_jiffies_ts - rq->irqload_ts; + if (p->sched_class->task_waking) + p->sched_class->task_waking(p); - if (nr_windows) { - if (nr_windows < 10) { - /* Decay CPU's irqload by 3/4 for each window. */ - rq->avg_irqload *= (3 * nr_windows); - rq->avg_irqload = div64_u64(rq->avg_irqload, - 4 * nr_windows); - } else { - rq->avg_irqload = 0; - } - rq->avg_irqload += rq->cur_irqload; - rq->cur_irqload = 0; + cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags); + if (task_cpu(p) != cpu) { + wake_flags |= WF_MIGRATED; + set_task_cpu(p, cpu); } +#endif /* CONFIG_SMP */ - rq->cur_irqload += delta; - rq->irqload_ts = cur_jiffies_ts; - raw_spin_unlock_irqrestore(&rq->lock, flags); -} + ttwu_queue(p, cpu); +stat: + ttwu_stat(p, cpu, wake_flags); +out: + raw_spin_unlock_irqrestore(&p->pi_lock, flags); -unsigned long __weak arch_get_cpu_efficiency(int cpu) -{ - return SCHED_LOAD_SCALE; + return success; } -static void init_cpu_efficiency(void) +/** + * try_to_wake_up_local - try to wake up a local task with rq lock held + * @p: the thread to be awakened + * + * Put @p on the run-queue if it's not already there. The caller must + * ensure that this_rq() is locked, @p is bound to this_rq() and not + * the current task. + */ +static void try_to_wake_up_local(struct task_struct *p) { - int i, efficiency; - unsigned int max = 0, min = UINT_MAX; + struct rq *rq = task_rq(p); - if (!sched_enable_hmp) + if (WARN_ON_ONCE(rq != this_rq()) || + WARN_ON_ONCE(p == current)) return; - for_each_possible_cpu(i) { - efficiency = arch_get_cpu_efficiency(i); - cpu_rq(i)->efficiency = efficiency; + lockdep_assert_held(&rq->lock); - if (efficiency > max) - max = efficiency; - if (efficiency < min) - min = efficiency; + if (!raw_spin_trylock(&p->pi_lock)) { + raw_spin_unlock(&rq->lock); + raw_spin_lock(&p->pi_lock); + raw_spin_lock(&rq->lock); } - BUG_ON(!max || !min); - - max_possible_efficiency = max; - min_possible_efficiency = min; -} - -static void reset_task_stats(struct task_struct *p) -{ - u32 sum = 0; + if (!(p->state & TASK_NORMAL)) + goto out; - if (exiting_task(p)) - sum = EXITING_TASK_MARKER; + if (!task_on_rq_queued(p)) + ttwu_activate(rq, p, ENQUEUE_WAKEUP); - memset(&p->ravg, 0, sizeof(struct ravg)); - /* Retain EXITING_TASK marker */ - p->ravg.sum_history[0] = sum; + ttwu_do_wakeup(rq, p, 0); + ttwu_stat(p, smp_processor_id(), 0); +out: + raw_spin_unlock(&p->pi_lock); } -static inline void mark_task_starting(struct task_struct *p) -{ - struct rq *rq = task_rq(p); - u64 wallclock = sched_clock(); - - if (!rq->window_start || sched_disable_window_stats) { - reset_task_stats(p); - return; - } - - p->ravg.mark_start = wallclock; -} - -static inline void set_window_start(struct rq *rq) +/** + * wake_up_process - Wake up a specific process + * @p: The process to be woken up. + * + * Attempt to wake up the nominated process and move it to the set of runnable + * processes. + * + * Return: 1 if the process was woken up, 0 if it was already running. + * + * It may be assumed that this function implies a write memory barrier before + * changing the task state if and only if any tasks are woken up. + */ +int wake_up_process(struct task_struct *p) { - int cpu = cpu_of(rq); - struct rq *sync_rq = cpu_rq(sync_cpu); - - if (rq->window_start || !sched_enable_hmp || - !sched_clock_initialized() || !sched_clock_cpu(cpu)) - return; - - if (cpu == sync_cpu) { - rq->window_start = sched_clock(); - } else { - raw_spin_unlock(&rq->lock); - double_rq_lock(rq, sync_rq); - rq->window_start = cpu_rq(sync_cpu)->window_start; -#ifdef CONFIG_SCHED_FREQ_INPUT - rq->curr_runnable_sum = rq->prev_runnable_sum = 0; -#endif - raw_spin_unlock(&sync_rq->lock); - } - - rq->curr->ravg.mark_start = rq->window_start; + WARN_ON(task_is_stopped_or_traced(p)); + return try_to_wake_up(p, TASK_NORMAL, 0); } +EXPORT_SYMBOL(wake_up_process); -static inline void migrate_sync_cpu(int cpu) +int wake_up_state(struct task_struct *p, unsigned int state) { - if (cpu == sync_cpu) - sync_cpu = smp_processor_id(); + return try_to_wake_up(p, state, 0); } -static void reset_all_task_stats(void) +/* + * This function clears the sched_dl_entity static params. + */ +void __dl_clear_params(struct task_struct *p) { - struct task_struct *g, *p; + struct sched_dl_entity *dl_se = &p->dl; - read_lock(&tasklist_lock); - do_each_thread(g, p) { - reset_task_stats(p); - } while_each_thread(g, p); - read_unlock(&tasklist_lock); + dl_se->dl_runtime = 0; + dl_se->dl_deadline = 0; + dl_se->dl_period = 0; + dl_se->flags = 0; + dl_se->dl_bw = 0; } /* - * sched_exit() - Set EXITING_TASK_MARKER in task's ravg.demand field - * - * Stop accounting (exiting) task's future cpu usage + * Perform scheduler related setup for a newly forked process p. + * p is forked by current. * - * We need this so that reset_all_windows_stats() can function correctly. - * reset_all_window_stats() depends on do_each_thread/for_each_thread task - * iterators to reset *all* task's statistics. Exiting tasks however become - * invisible to those iterators. sched_exit() is called on a exiting task prior - * to being removed from task_list, which will let reset_all_window_stats() - * function correctly. + * __sched_fork() is basic setup used by init_idle() too: */ -void sched_exit(struct task_struct *p) +static void __sched_fork(unsigned long clone_flags, struct task_struct *p) { - unsigned long flags; - int cpu = get_cpu(); - struct rq *rq = cpu_rq(cpu); - u64 wallclock; + p->on_rq = 0; - raw_spin_lock_irqsave(&rq->lock, flags); - /* rq->curr == p */ - wallclock = sched_clock(); - update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); - dequeue_task(rq, p, 0); - reset_task_stats(p); - p->ravg.mark_start = wallclock; - p->ravg.sum_history[0] = EXITING_TASK_MARKER; - enqueue_task(rq, p, 0); - raw_spin_unlock_irqrestore(&rq->lock, flags); + p->se.on_rq = 0; + p->se.exec_start = 0; + p->se.sum_exec_runtime = 0; + p->se.prev_sum_exec_runtime = 0; + p->se.nr_migrations = 0; + p->se.vruntime = 0; + INIT_LIST_HEAD(&p->se.group_node); - put_cpu(); -} +#ifdef CONFIG_SCHEDSTATS + memset(&p->se.statistics, 0, sizeof(p->se.statistics)); +#endif -static void disable_window_stats(void) -{ - unsigned long flags; - int i; + RB_CLEAR_NODE(&p->dl.rb_node); + hrtimer_init(&p->dl.dl_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + __dl_clear_params(p); - local_irq_save(flags); - for_each_possible_cpu(i) - raw_spin_lock(&cpu_rq(i)->lock); + INIT_LIST_HEAD(&p->rt.run_list); - sched_disable_window_stats = 1; +#ifdef CONFIG_PREEMPT_NOTIFIERS + INIT_HLIST_HEAD(&p->preempt_notifiers); +#endif - for_each_possible_cpu(i) - raw_spin_unlock(&cpu_rq(i)->lock); +#ifdef CONFIG_NUMA_BALANCING + if (p->mm && atomic_read(&p->mm->mm_users) == 1) { + p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); + p->mm->numa_scan_seq = 0; + } - local_irq_restore(flags); -} + if (clone_flags & CLONE_VM) + p->numa_preferred_nid = current->numa_preferred_nid; + else + p->numa_preferred_nid = -1; -/* Called with all cpu's rq->lock held */ -static void enable_window_stats(void) -{ - sched_disable_window_stats = 0; + p->node_stamp = 0ULL; + p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0; + p->numa_scan_period = sysctl_numa_balancing_scan_delay; + p->numa_work.next = &p->numa_work; + p->numa_faults_memory = NULL; + p->numa_faults_buffer_memory = NULL; + p->last_task_numa_placement = 0; + p->last_sum_exec_runtime = 0; + INIT_LIST_HEAD(&p->numa_entry); + p->numa_group = NULL; +#endif /* CONFIG_NUMA_BALANCING */ } -enum reset_reason_code { - WINDOW_CHANGE, - POLICY_CHANGE, - ACCOUNT_WAIT_TIME_CHANGE, - HIST_SIZE_CHANGE, - MIGRATION_FIXUP_CHANGE, - FREQ_ACCOUNT_WAIT_TIME_CHANGE -}; - -const char *sched_window_reset_reasons[] = { - "WINDOW_CHANGE", - "POLICY_CHANGE", - "ACCOUNT_WAIT_TIME_CHANGE", - "HIST_SIZE_CHANGE", - "MIGRATION_FIXUP_CHANGE", - "FREQ_ACCOUNT_WAIT_TIME_CHANGE"}; - -/* Called with IRQs enabled */ -void reset_all_window_stats(u64 window_start, unsigned int window_size) +#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_SCHED_DEBUG +void set_numabalancing_state(bool enabled) { - int cpu; - unsigned long flags; - u64 start_ts = sched_clock(); - int reason = WINDOW_CHANGE; - unsigned int old = 0, new = 0; - unsigned int old_window_size = sched_ravg_window; - - disable_window_stats(); - - reset_all_task_stats(); - - local_irq_save(flags); - - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - raw_spin_lock(&rq->lock); - } - - if (window_size) { - sched_ravg_window = window_size * TICK_NSEC; - set_hmp_defaults(); - } + if (enabled) + sched_feat_set("NUMA"); + else + sched_feat_set("NO_NUMA"); +} +#else +__read_mostly bool numabalancing_enabled; - enable_window_stats(); +void set_numabalancing_state(bool enabled) +{ + numabalancing_enabled = enabled; +} +#endif /* CONFIG_SCHED_DEBUG */ - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); +#ifdef CONFIG_PROC_SYSCTL +int sysctl_numa_balancing(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + struct ctl_table t; + int err; + int state = numabalancing_enabled; - if (window_start) { - u32 mostly_idle_load = rq->mostly_idle_load; + if (write && !capable(CAP_SYS_ADMIN)) + return -EPERM; - rq->window_start = window_start; - rq->mostly_idle_load = div64_u64((u64)mostly_idle_load * - (u64)sched_ravg_window, (u64)old_window_size); - } -#ifdef CONFIG_SCHED_FREQ_INPUT - rq->curr_runnable_sum = rq->prev_runnable_sum = 0; + t = *table; + t.data = &state; + err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); + if (err < 0) + return err; + if (write) + set_numabalancing_state(state); + return err; +} #endif - reset_cpu_hmp_stats(cpu, 1); - - fixup_nr_big_small_task(cpu, 0); - } - - if (sched_window_stats_policy != sysctl_sched_window_stats_policy) { - reason = POLICY_CHANGE; - old = sched_window_stats_policy; - new = sysctl_sched_window_stats_policy; - sched_window_stats_policy = sysctl_sched_window_stats_policy; - } else if (sched_account_wait_time != sysctl_sched_account_wait_time) { - reason = ACCOUNT_WAIT_TIME_CHANGE; - old = sched_account_wait_time; - new = sysctl_sched_account_wait_time; - sched_account_wait_time = sysctl_sched_account_wait_time; - } else if (sched_ravg_hist_size != sysctl_sched_ravg_hist_size) { - reason = HIST_SIZE_CHANGE; - old = sched_ravg_hist_size; - new = sysctl_sched_ravg_hist_size; - sched_ravg_hist_size = sysctl_sched_ravg_hist_size; - } -#ifdef CONFIG_SCHED_FREQ_INPUT - else if (sched_migration_fixup != sysctl_sched_migration_fixup) { - reason = MIGRATION_FIXUP_CHANGE; - old = sched_migration_fixup; - new = sysctl_sched_migration_fixup; - sched_migration_fixup = sysctl_sched_migration_fixup; - } else if (sched_freq_account_wait_time != - sysctl_sched_freq_account_wait_time) { - reason = FREQ_ACCOUNT_WAIT_TIME_CHANGE; - old = sched_freq_account_wait_time; - new = sysctl_sched_freq_account_wait_time; - sched_freq_account_wait_time = - sysctl_sched_freq_account_wait_time; - } #endif - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - raw_spin_unlock(&rq->lock); - } - - local_irq_restore(flags); - - trace_sched_reset_all_window_stats(window_start, window_size, - sched_clock() - start_ts, reason, old, new); -} - -#ifdef CONFIG_SCHED_FREQ_INPUT - -unsigned long sched_get_busy(int cpu) +/* + * fork()/clone()-time setup: + */ +int sched_fork(unsigned long clone_flags, struct task_struct *p) { unsigned long flags; - struct rq *rq = cpu_rq(cpu); - u64 load; + int cpu = get_cpu(); + __sched_fork(clone_flags, p); /* - * This function could be called in timer context, and the - * current task may have been executing for a long time. Ensure - * that the window stats are current by doing an update. + * We mark the process as running here. This guarantees that + * nobody will actually run it, and a signal or other external + * event cannot wake it up and insert it on the runqueue either. */ - raw_spin_lock_irqsave(&rq->lock, flags); - update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), 0); - load = rq->old_busy_time = rq->prev_runnable_sum; - raw_spin_unlock_irqrestore(&rq->lock, flags); + p->state = TASK_RUNNING; /* - * Scale load in reference to rq->max_possible_freq. - * - * Note that scale_load_to_cpu() scales load in reference to - * rq->max_freq + * Make sure we do not leak PI boosting priority to the child. */ - load = scale_load_to_cpu(load, cpu); - load = div64_u64(load * (u64)rq->max_freq, (u64)rq->max_possible_freq); - load = div64_u64(load, NSEC_PER_USEC); - - trace_sched_get_busy(cpu, load); - - return load; -} - -void sched_set_io_is_busy(int val) -{ - sched_io_is_busy = val; -} - -int sched_set_window(u64 window_start, unsigned int window_size) -{ - u64 now, cur_jiffies, jiffy_sched_clock; - s64 ws; - unsigned long flags; - - if (sched_use_pelt || - (window_size * TICK_NSEC < MIN_SCHED_RAVG_WINDOW)) - return -EINVAL; + p->prio = current->normal_prio; - mutex_lock(&policy_mutex); + /* + * Revert to default priority/policy on fork if requested. + */ + if (unlikely(p->sched_reset_on_fork)) { + if (task_has_dl_policy(p) || task_has_rt_policy(p)) { + p->policy = SCHED_NORMAL; + p->static_prio = NICE_TO_PRIO(0); + p->rt_priority = 0; + } else if (PRIO_TO_NICE(p->static_prio) < 0) + p->static_prio = NICE_TO_PRIO(0); - /* Get a consistent view of sched_clock, jiffies, and the time - * since the last jiffy (based on last_jiffies_update). */ - local_irq_save(flags); - cur_jiffies = jiffy_to_sched_clock(&now, &jiffy_sched_clock); - local_irq_restore(flags); + p->prio = p->normal_prio = __normal_prio(p); + set_load_weight(p); - /* translate window_start from jiffies to nanoseconds */ - ws = (window_start - cur_jiffies); /* jiffy difference */ - ws *= TICK_NSEC; - ws += jiffy_sched_clock; + /* + * We don't need the reset flag anymore after the fork. It has + * fulfilled its duty: + */ + p->sched_reset_on_fork = 0; + } - /* roll back calculated window start so that it is in - * the past (window stats must have a current window) */ - while (ws > now) - ws -= (window_size * TICK_NSEC); + if (dl_prio(p->prio)) { + put_cpu(); + return -EAGAIN; + } else if (rt_prio(p->prio)) { + p->sched_class = &rt_sched_class; + } else { + p->sched_class = &fair_sched_class; + } - BUG_ON(sched_clock() < ws); + if (p->sched_class->task_fork) + p->sched_class->task_fork(p); - reset_all_window_stats(ws, window_size); + /* + * The child is not yet in the pid-hash so no cgroup attach races, + * and the cgroup is pinned to this child due to cgroup_fork() + * is ran before sched_fork(). + * + * Silence PROVE_RCU. + */ + raw_spin_lock_irqsave(&p->pi_lock, flags); + set_task_cpu(p, cpu); + raw_spin_unlock_irqrestore(&p->pi_lock, flags); - mutex_unlock(&policy_mutex); +#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) + if (likely(sched_info_on())) + memset(&p->sched_info, 0, sizeof(p->sched_info)); +#endif +#if defined(CONFIG_SMP) + p->on_cpu = 0; +#endif + task_thread_info(p)->preempt_count = 1; +#ifdef CONFIG_SMP + plist_node_init(&p->pushable_tasks, MAX_PRIO); + RB_CLEAR_NODE(&p->pushable_dl_tasks); +#endif + put_cpu(); return 0; } -static void fixup_busy_time(struct task_struct *p, int new_cpu) +unsigned long to_ratio(u64 period, u64 runtime) { - struct rq *src_rq = task_rq(p); - struct rq *dest_rq = cpu_rq(new_cpu); - u64 wallclock; - - if (!sched_enable_hmp || !sched_migration_fixup || - exiting_task(p) || (!p->on_rq && p->state != TASK_WAKING)) - return; - - if (p->state == TASK_WAKING) - double_rq_lock(src_rq, dest_rq); - - if (sched_disable_window_stats) - goto done; - - wallclock = sched_clock(); - - update_task_ravg(task_rq(p)->curr, task_rq(p), - TASK_UPDATE, - wallclock, 0); - update_task_ravg(dest_rq->curr, dest_rq, - TASK_UPDATE, wallclock, 0); - - /* - * In case of migration of task on runqueue, on_rq =1, - * however its load is removed from its runqueue. - * update_task_ravg() below can update its demand, which - * will require its load on runqueue to be adjusted to - * reflect new demand. Restore load temporarily for such - * task on its runqueue - */ - if (p->on_rq) - p->sched_class->inc_hmp_sched_stats(src_rq, p); - - update_task_ravg(p, task_rq(p), TASK_MIGRATE, - wallclock, 0); + if (runtime == RUNTIME_INF) + return 1ULL << 20; /* - * Remove task's load from rq as its now migrating to - * another cpu. + * Doing this here saves a lot of checks in all + * the calling paths, and returning zero seems + * safe for them anyway. */ - if (p->on_rq) - p->sched_class->dec_hmp_sched_stats(src_rq, p); - - if (p->ravg.curr_window) { - src_rq->curr_runnable_sum -= p->ravg.curr_window; - dest_rq->curr_runnable_sum += p->ravg.curr_window; - } - - if (p->ravg.prev_window) { - src_rq->prev_runnable_sum -= p->ravg.prev_window; - dest_rq->prev_runnable_sum += p->ravg.prev_window; - } - - BUG_ON((s64)src_rq->prev_runnable_sum < 0); - BUG_ON((s64)src_rq->curr_runnable_sum < 0); - - trace_sched_migration_update_sum(src_rq, p); - trace_sched_migration_update_sum(dest_rq, p); + if (period == 0) + return 0; -done: - if (p->state == TASK_WAKING) - double_rq_unlock(src_rq, dest_rq); + return div64_u64(runtime << 20, period); } -#else - -static inline void fixup_busy_time(struct task_struct *p, int new_cpu) { } - -static inline int -heavy_task_wakeup(struct task_struct *p, struct rq *rq, int event) +#ifdef CONFIG_SMP +inline struct dl_bw *dl_bw_of(int i) { - return 0; + rcu_lockdep_assert(rcu_read_lock_sched_held(), + "sched RCU must be held"); + return &cpu_rq(i)->rd->dl_bw; } -#endif /* CONFIG_SCHED_FREQ_INPUT */ - -/* Keep track of max/min capacity possible across CPUs "currently" */ -static void __update_min_max_capacity(void) +static inline int dl_bw_cpus(int i) { - int i; - int max = 0, min = INT_MAX; + struct root_domain *rd = cpu_rq(i)->rd; + int cpus = 0; - for_each_online_cpu(i) { - if (cpu_rq(i)->capacity > max) - max = cpu_rq(i)->capacity; - if (cpu_rq(i)->capacity < min) - min = cpu_rq(i)->capacity; - } + rcu_lockdep_assert(rcu_read_lock_sched_held(), + "sched RCU must be held"); + for_each_cpu_and(i, rd->span, cpu_active_mask) + cpus++; - max_capacity = max; - min_capacity = min; + return cpus; } - -static void update_min_max_capacity(void) +#else +inline struct dl_bw *dl_bw_of(int i) { - int i; - - for_each_possible_cpu(i) - raw_spin_lock(&cpu_rq(i)->lock); - - __update_min_max_capacity(); + return &cpu_rq(i)->dl.dl_bw; +} - for_each_possible_cpu(i) - raw_spin_unlock(&cpu_rq(i)->lock); +static inline int dl_bw_cpus(int i) +{ + return 1; } +#endif -/* - * Return 'capacity' of a cpu in reference to "least" efficient cpu, such that - * least efficient cpu gets capacity of 1024 - */ -unsigned long capacity_scale_cpu_efficiency(int cpu) +static inline +void __dl_clear(struct dl_bw *dl_b, u64 tsk_bw) { - return (1024 * cpu_rq(cpu)->efficiency) / min_possible_efficiency; + dl_b->total_bw -= tsk_bw; } -/* - * Return 'capacity' of a cpu in reference to cpu with lowest max_freq - * (min_max_freq), such that one with lowest max_freq gets capacity of 1024. - */ -unsigned long capacity_scale_cpu_freq(int cpu) +static inline +void __dl_add(struct dl_bw *dl_b, u64 tsk_bw) { - return (1024 * cpu_rq(cpu)->max_freq) / min_max_freq; + dl_b->total_bw += tsk_bw; } -/* - * Return load_scale_factor of a cpu in reference to "most" efficient cpu, so - * that "most" efficient cpu gets a load_scale_factor of 1 - */ -static inline unsigned long load_scale_cpu_efficiency(int cpu) +static inline +bool __dl_overflow(struct dl_bw *dl_b, int cpus, u64 old_bw, u64 new_bw) { - return DIV_ROUND_UP(1024 * max_possible_efficiency, - cpu_rq(cpu)->efficiency); + return dl_b->bw != -1 && + dl_b->bw * cpus < dl_b->total_bw - old_bw + new_bw; } /* - * Return load_scale_factor of a cpu in reference to cpu with best max_freq - * (max_possible_freq), so that one with best max_freq gets a load_scale_factor - * of 1. + * We must be sure that accepting a new task (or allowing changing the + * parameters of an existing one) is consistent with the bandwidth + * constraints. If yes, this function also accordingly updates the currently + * allocated bandwidth to reflect the new situation. + * + * This function is called while holding p's rq->lock. */ -static inline unsigned long load_scale_cpu_freq(int cpu) +static int dl_overflow(struct task_struct *p, int policy, + const struct sched_attr *attr) { - return DIV_ROUND_UP(1024 * max_possible_freq, cpu_rq(cpu)->max_freq); -} -static int compute_capacity(int cpu) -{ - int capacity = 1024; + struct dl_bw *dl_b = dl_bw_of(task_cpu(p)); + u64 period = attr->sched_period ?: attr->sched_deadline; + u64 runtime = attr->sched_runtime; + u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0; + int cpus, err = -1; - capacity *= capacity_scale_cpu_efficiency(cpu); - capacity >>= 10; + if (new_bw == p->dl.dl_bw) + return 0; - capacity *= capacity_scale_cpu_freq(cpu); - capacity >>= 10; + /* + * Either if a task, enters, leave, or stays -deadline but changes + * its parameters, we may need to update accordingly the total + * allocated bandwidth of the container. + */ + raw_spin_lock(&dl_b->lock); + cpus = dl_bw_cpus(task_cpu(p)); + if (dl_policy(policy) && !task_has_dl_policy(p) && + !__dl_overflow(dl_b, cpus, 0, new_bw)) { + __dl_add(dl_b, new_bw); + err = 0; + } else if (dl_policy(policy) && task_has_dl_policy(p) && + !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) { + __dl_clear(dl_b, p->dl.dl_bw); + __dl_add(dl_b, new_bw); + err = 0; + } else if (!dl_policy(policy) && task_has_dl_policy(p)) { + __dl_clear(dl_b, p->dl.dl_bw); + err = 0; + } + raw_spin_unlock(&dl_b->lock); - return capacity; + return err; } -static int compute_load_scale_factor(int cpu) +extern void init_dl_bw(struct dl_bw *dl_b); + +/* + * wake_up_new_task - wake up a newly created task for the first time. + * + * This function will do some initial scheduler statistics housekeeping + * that must be done for every newly created context, then puts the task + * on the runqueue and wakes it. + */ +void wake_up_new_task(struct task_struct *p) { - int load_scale = 1024; + unsigned long flags; + struct rq *rq; + raw_spin_lock_irqsave(&p->pi_lock, flags); + /* Initialize new task's runnable average */ + init_task_runnable_average(p); +#ifdef CONFIG_SMP /* - * load_scale_factor accounts for the fact that task load - * is in reference to "best" performing cpu. Task's load will need to be - * scaled (up) by a factor to determine suitability to be placed on a - * (little) cpu. + * Fork balancing, do it here and not earlier because: + * - cpus_allowed can change in the fork path + * - any previously selected cpu might disappear through hotplug */ - load_scale *= load_scale_cpu_efficiency(cpu); - load_scale >>= 10; - - load_scale *= load_scale_cpu_freq(cpu); - load_scale >>= 10; + set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0)); +#endif - return load_scale; + rq = __task_rq_lock(p); + activate_task(rq, p, ENQUEUE_WAKEUP_NEW); + p->on_rq = TASK_ON_RQ_QUEUED; + trace_sched_wakeup_new(p, true); + check_preempt_curr(rq, p, WF_FORK); +#ifdef CONFIG_SMP + if (p->sched_class->task_woken) + p->sched_class->task_woken(rq, p); +#endif + task_rq_unlock(rq, p, &flags); } -static int cpufreq_notifier_policy(struct notifier_block *nb, - unsigned long val, void *data) -{ - struct cpufreq_policy *policy = (struct cpufreq_policy *)data; - int i, update_max = 0; - u64 highest_mpc = 0, highest_mplsf = 0; - const struct cpumask *cpus = policy->related_cpus; - unsigned int orig_min_max_freq = min_max_freq; - unsigned int orig_max_possible_freq = max_possible_freq; - /* Initialized to policy->max in case policy->related_cpus is empty! */ - unsigned int orig_max_freq = policy->max; - - if (val != CPUFREQ_NOTIFY && val != CPUFREQ_REMOVE_POLICY) - return 0; - - if (val == CPUFREQ_REMOVE_POLICY) { - update_min_max_capacity(); - return 0; - } - - for_each_cpu(i, policy->related_cpus) { - cpumask_copy(&cpu_rq(i)->freq_domain_cpumask, - policy->related_cpus); - orig_max_freq = cpu_rq(i)->max_freq; - cpu_rq(i)->min_freq = policy->min; - cpu_rq(i)->max_freq = policy->max; - cpu_rq(i)->max_possible_freq = policy->cpuinfo.max_freq; - } - - max_possible_freq = max(max_possible_freq, policy->cpuinfo.max_freq); - if (min_max_freq == 1) - min_max_freq = UINT_MAX; - min_max_freq = min(min_max_freq, policy->cpuinfo.max_freq); - BUG_ON(!min_max_freq); - BUG_ON(!policy->max); - - /* - * A changed min_max_freq or max_possible_freq (possible during bootup) - * needs to trigger re-computation of load_scale_factor and capacity for - * all possible cpus (even those offline). It also needs to trigger - * re-computation of nr_big/small_task count on all online cpus. - * - * A changed rq->max_freq otoh needs to trigger re-computation of - * load_scale_factor and capacity for just the cluster of cpus involved. - * Since small task definition depends on max_load_scale_factor, a - * changed load_scale_factor of one cluster could influence small_task - * classification of tasks in another cluster. Hence a changed - * rq->max_freq will need to trigger re-computation of nr_big/small_task - * count on all online cpus. - * - * While it should be sufficient for nr_big/small_tasks to be - * re-computed for only online cpus, we have inadequate context - * information here (in policy notifier) with regard to hotplug-safety - * context in which notification is issued. As a result, we can't use - * get_online_cpus() here, as it can lead to deadlock. Until cpufreq is - * fixed up to issue notification always in hotplug-safe context, - * re-compute nr_big/small_task for all possible cpus. - */ +#ifdef CONFIG_PREEMPT_NOTIFIERS - if (orig_min_max_freq != min_max_freq || - orig_max_possible_freq != max_possible_freq) { - cpus = cpu_possible_mask; - update_max = 1; - } +/** + * preempt_notifier_register - tell me when current is being preempted & rescheduled + * @notifier: notifier struct to register + */ +void preempt_notifier_register(struct preempt_notifier *notifier) +{ + hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); +} +EXPORT_SYMBOL_GPL(preempt_notifier_register); - /* - * Changed load_scale_factor can trigger reclassification of tasks as - * big or small. Make this change "atomic" so that tasks are accounted - * properly due to changed load_scale_factor - */ - pre_big_small_task_count_change(cpu_possible_mask); - for_each_cpu(i, cpus) { - struct rq *rq = cpu_rq(i); +/** + * preempt_notifier_unregister - no longer interested in preemption notifications + * @notifier: notifier struct to unregister + * + * This is safe to call from within a preemption notifier. + */ +void preempt_notifier_unregister(struct preempt_notifier *notifier) +{ + hlist_del(¬ifier->link); +} +EXPORT_SYMBOL_GPL(preempt_notifier_unregister); - rq->capacity = compute_capacity(i); - rq->load_scale_factor = compute_load_scale_factor(i); +static void fire_sched_in_preempt_notifiers(struct task_struct *curr) +{ + struct preempt_notifier *notifier; - if (update_max) { - u64 mpc, mplsf; + hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) + notifier->ops->sched_in(notifier, raw_smp_processor_id()); +} - mpc = div_u64(((u64) rq->capacity) * - rq->max_possible_freq, rq->max_freq); - rq->max_possible_capacity = (int) mpc; +static void +fire_sched_out_preempt_notifiers(struct task_struct *curr, + struct task_struct *next) +{ + struct preempt_notifier *notifier; - mplsf = div_u64(((u64) rq->load_scale_factor) * - rq->max_possible_freq, rq->max_freq); + hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) + notifier->ops->sched_out(notifier, next); +} - if (mpc > highest_mpc) { - highest_mpc = mpc; - cpumask_clear(&mpc_mask); - cpumask_set_cpu(i, &mpc_mask); - } else if (mpc == highest_mpc) { - cpumask_set_cpu(i, &mpc_mask); - } +#else /* !CONFIG_PREEMPT_NOTIFIERS */ - if (mplsf > highest_mplsf) - highest_mplsf = mplsf; - } - } +static void fire_sched_in_preempt_notifiers(struct task_struct *curr) +{ +} - if (update_max) { - max_possible_capacity = highest_mpc; - max_load_scale_factor = highest_mplsf; - } +static void +fire_sched_out_preempt_notifiers(struct task_struct *curr, + struct task_struct *next) +{ +} - __update_min_max_capacity(); - post_big_small_task_count_change(cpu_possible_mask); +#endif /* CONFIG_PREEMPT_NOTIFIERS */ - return 0; +/** + * prepare_task_switch - prepare to switch tasks + * @rq: the runqueue preparing to switch + * @prev: the current task that is being switched out + * @next: the task we are going to switch to. + * + * This is called with the rq lock held and interrupts off. It must + * be paired with a subsequent finish_task_switch after the context + * switch. + * + * prepare_task_switch sets up locking and calls architecture specific + * hooks. + */ +static inline void +prepare_task_switch(struct rq *rq, struct task_struct *prev, + struct task_struct *next) +{ + trace_sched_switch(prev, next); + sched_info_switch(rq, prev, next); + perf_event_task_sched_out(prev, next); + fire_sched_out_preempt_notifiers(prev, next); + prepare_lock_switch(rq, next); + prepare_arch_switch(next); } -static int cpufreq_notifier_trans(struct notifier_block *nb, - unsigned long val, void *data) +/** + * finish_task_switch - clean up after a task-switch + * @rq: runqueue associated with task-switch + * @prev: the thread we just switched away from. + * + * finish_task_switch must be called after the context switch, paired + * with a prepare_task_switch call before the context switch. + * finish_task_switch will reconcile locking set up by prepare_task_switch, + * and do any other architecture-specific cleanup actions. + * + * Note that we may have delayed dropping an mm in context_switch(). If + * so, we finish that here outside of the runqueue lock. (Doing it + * with the lock held can cause deadlocks; see schedule() for + * details.) + */ +static void finish_task_switch(struct rq *rq, struct task_struct *prev) + __releases(rq->lock) { - struct cpufreq_freqs *freq = (struct cpufreq_freqs *)data; - unsigned int cpu = freq->cpu, new_freq = freq->new; - unsigned long flags; - int i; + struct mm_struct *mm = rq->prev_mm; + long prev_state; - if (val != CPUFREQ_POSTCHANGE) - return 0; + rq->prev_mm = NULL; - BUG_ON(!new_freq); + /* + * A task struct has one reference for the use as "current". + * If a task dies, then it sets TASK_DEAD in tsk->state and calls + * schedule one last time. The schedule call will never return, and + * the scheduled task must drop that reference. + * The test for TASK_DEAD must occur while the runqueue locks are + * still held, otherwise prev could be scheduled on another cpu, die + * there before we look at prev->state, and then the reference would + * be dropped twice. + * Manfred Spraul + */ + prev_state = prev->state; + vtime_task_switch(prev); + finish_arch_switch(prev); + perf_event_task_sched_in(prev, current); + finish_lock_switch(rq, prev); + finish_arch_post_lock_switch(); - if (cpu_rq(cpu)->cur_freq == new_freq) - return 0; + fire_sched_in_preempt_notifiers(current); + if (mm) + mmdrop(mm); + if (unlikely(prev_state == TASK_DEAD)) { + if (prev->sched_class->task_dead) + prev->sched_class->task_dead(prev); - for_each_cpu(i, &cpu_rq(cpu)->freq_domain_cpumask) { - struct rq *rq = cpu_rq(i); - raw_spin_lock_irqsave(&rq->lock, flags); - update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), 0); - rq->cur_freq = new_freq; - raw_spin_unlock_irqrestore(&rq->lock, flags); + /* + * Remove function-return probe instances associated with this + * task and put them back on the free list. + */ + kprobe_flush_task(prev); + put_task_struct(prev); } - return 0; + tick_nohz_task_switch(current); } -static struct notifier_block notifier_policy_block = { - .notifier_call = cpufreq_notifier_policy -}; - -static struct notifier_block notifier_trans_block = { - .notifier_call = cpufreq_notifier_trans -}; +#ifdef CONFIG_SMP -static int register_sched_callback(void) +/* rq->lock is NOT held, but preemption is disabled */ +static inline void post_schedule(struct rq *rq) { - int ret; - - if (!sched_enable_hmp) - return 0; - - ret = cpufreq_register_notifier(¬ifier_policy_block, - CPUFREQ_POLICY_NOTIFIER); + if (rq->post_schedule) { + unsigned long flags; - if (!ret) - ret = cpufreq_register_notifier(¬ifier_trans_block, - CPUFREQ_TRANSITION_NOTIFIER); + raw_spin_lock_irqsave(&rq->lock, flags); + if (rq->curr->sched_class->post_schedule) + rq->curr->sched_class->post_schedule(rq); + raw_spin_unlock_irqrestore(&rq->lock, flags); - return 0; + rq->post_schedule = 0; + } } -/* - * cpufreq callbacks can be registered at core_initcall or later time. - * Any registration done prior to that is "forgotten" by cpufreq. See - * initialization of variable init_cpufreq_transition_notifier_list_called - * for further information. - */ -core_initcall(register_sched_callback); +#else -static u64 orig_mark_start(struct task_struct *p) +static inline void post_schedule(struct rq *rq) { - return p->ravg.mark_start; } +>>>>>>> 374fbbf4d13... Backport EAS scheduler from 3.18 -static void restore_orig_mark_start(struct task_struct *p, u64 mark_start) -{ - p->ravg.mark_start = mark_start; -} +#endif -/* - * Note down when task started running on a cpu. This information will be handy - * to avoid "too" frequent task migrations for a running task on account of - * power. +/** + * schedule_tail - first thing a freshly forked thread must call. + * @prev: the thread we just switched away from. */ -static inline void note_run_start(struct task_struct *p, u64 wallclock) +asmlinkage void schedule_tail(struct task_struct *prev) + __releases(rq->lock) { - p->run_start = wallclock; -} + struct rq *rq = this_rq(); -#else /* CONFIG_SCHED_HMP */ + finish_task_switch(rq, prev); -static inline void fixup_busy_time(struct task_struct *p, int new_cpu) { } + /* + * FIXME: do we need to worry about rq being invalidated by the + * task_switch? + */ + post_schedule(rq); -static inline int -heavy_task_wakeup(struct task_struct *p, struct rq *rq, int event) -{ - return 0; + if (current->set_child_tid) + put_user(task_pid_vnr(current), current->set_child_tid); } +/* + * context_switch - switch to the new MM and the new + * thread's register state. + */ static inline void -update_task_ravg(struct task_struct *p, struct rq *rq, - int event, u64 wallclock, u64 irqtime) +context_switch(struct rq *rq, struct task_struct *prev, + struct task_struct *next) { -} - -static inline void init_cpu_efficiency(void) {} + struct mm_struct *mm, *oldmm; -static inline void mark_task_starting(struct task_struct *p) {} + prepare_task_switch(rq, prev, next); -static inline void set_window_start(struct rq *rq) {} + mm = next->mm; + oldmm = prev->active_mm; + /* + * For paravirt, this is coupled with an exit in switch_to to + * combine the page table reload and the switch backend into + * one hypercall. + */ + arch_start_context_switch(prev); -static inline void migrate_sync_cpu(int cpu) {} + if (!mm) { + next->active_mm = oldmm; + atomic_inc(&oldmm->mm_count); + enter_lazy_tlb(oldmm, next); + } else + switch_mm(oldmm, mm, next); -static inline u64 orig_mark_start(struct task_struct *p) { return 0; } + if (!prev->mm) { + prev->active_mm = NULL; + rq->prev_mm = oldmm; + } + /* + * Since the runqueue lock will be released by the next + * task (which is an invalid locking op but in the case + * of the scheduler it's an obvious special-case), so we + * do an early lockdep release here: + */ + spin_release(&rq->lock.dep_map, 1, _THIS_IP_); -static inline void -restore_orig_mark_start(struct task_struct *p, u64 mark_start) -{ -} - -static inline void note_run_start(struct task_struct *p, u64 wallclock) { } - -#endif /* CONFIG_SCHED_HMP */ - -#ifdef CONFIG_SMP -void set_task_cpu(struct task_struct *p, unsigned int new_cpu) -{ -#ifdef CONFIG_SCHED_DEBUG - /* - * We should never call set_task_cpu() on a blocked task, - * ttwu() will sort out the placement. - */ - WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING && - !(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE)); - -#ifdef CONFIG_LOCKDEP - /* - * The caller should hold either p->pi_lock or rq->lock, when changing - * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. - * - * sched_move_task() holds both and thus holding either pins the cgroup, - * see task_group(). - * - * Furthermore, all task_rq users should acquire both locks, see - * task_rq_lock(). - */ - WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || - lockdep_is_held(&task_rq(p)->lock))); -#endif -#endif - - trace_sched_migrate_task(p, new_cpu, pct_task_load(p)); - - note_run_start(p, sched_clock()); - - if (task_cpu(p) != new_cpu) { - struct task_migration_notifier tmn; - - if (p->sched_class->migrate_task_rq) - p->sched_class->migrate_task_rq(p, new_cpu); - p->se.nr_migrations++; - perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0); - - tmn.task = p; - tmn.from_cpu = task_cpu(p); - tmn.to_cpu = new_cpu; - - atomic_notifier_call_chain(&task_migration_notifier, 0, &tmn); - - fixup_busy_time(p, new_cpu); - } - - __set_task_cpu(p, new_cpu); -} - -struct migration_arg { - struct task_struct *task; - int dest_cpu; -}; - -static int migration_cpu_stop(void *data); - -/* - * wait_task_inactive - wait for a thread to unschedule. - * - * If @match_state is nonzero, it's the @p->state value just checked and - * not expected to change. If it changes, i.e. @p might have woken up, - * then return zero. When we succeed in waiting for @p to be off its CPU, - * we return a positive number (its total switch count). If a second call - * a short while later returns the same number, the caller can be sure that - * @p has remained unscheduled the whole time. - * - * The caller must ensure that the task *will* unschedule sometime soon, - * else this function might spin for a *long* time. This function can't - * be called with interrupts off, or it may introduce deadlock with - * smp_call_function() if an IPI is sent by the same process we are - * waiting to become inactive. - */ -unsigned long wait_task_inactive(struct task_struct *p, long match_state) -{ - unsigned long flags; - int running, on_rq; - unsigned long ncsw; - struct rq *rq; - - for (;;) { - /* - * We do the initial early heuristics without holding - * any task-queue locks at all. We'll only try to get - * the runqueue lock when things look like they will - * work out! - */ - rq = task_rq(p); - - /* - * If the task is actively running on another CPU - * still, just relax and busy-wait without holding - * any locks. - * - * NOTE! Since we don't hold any locks, it's not - * even sure that "rq" stays as the right runqueue! - * But we don't care, since "task_running()" will - * return false if the runqueue has changed and p - * is actually now running somewhere else! - */ - while (task_running(rq, p)) { - if (match_state && unlikely(p->state != match_state)) - return 0; - cpu_relax(); - } - - /* - * Ok, time to look more closely! We need the rq - * lock now, to be *sure*. If we're wrong, we'll - * just go back and repeat. - */ - rq = task_rq_lock(p, &flags); - trace_sched_wait_task(p); - running = task_running(rq, p); - on_rq = p->on_rq; - ncsw = 0; - if (!match_state || p->state == match_state) - ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ - task_rq_unlock(rq, p, &flags); - - /* - * If it changed from the expected state, bail out now. - */ - if (unlikely(!ncsw)) - break; - - /* - * Was it really running after all now that we - * checked with the proper locks actually held? - * - * Oops. Go back and try again.. - */ - if (unlikely(running)) { - cpu_relax(); - continue; - } - - /* - * It's not enough that it's not actively running, - * it must be off the runqueue _entirely_, and not - * preempted! - * - * So if it was still runnable (but just not actively - * running right now), it's preempted, and we should - * yield - it could be a while. - */ - if (unlikely(on_rq)) { - ktime_t to = ktime_set(0, NSEC_PER_MSEC); - - set_current_state(TASK_UNINTERRUPTIBLE); - schedule_hrtimeout(&to, HRTIMER_MODE_REL); - continue; - } - - /* - * Ahh, all good. It wasn't running, and it wasn't - * runnable, which means that it will never become - * running in the future either. We're all done! - */ - break; - } - - return ncsw; -} - -/*** - * kick_process - kick a running thread to enter/exit the kernel - * @p: the to-be-kicked thread - * - * Cause a process which is running on another CPU to enter - * kernel-mode, without any delay. (to get signals handled.) - * - * NOTE: this function doesn't have to take the runqueue lock, - * because all it wants to ensure is that the remote task enters - * the kernel. If the IPI races and the task has been migrated - * to another CPU then no harm is done and the purpose has been - * achieved as well. - */ -void kick_process(struct task_struct *p) -{ - int cpu; - - preempt_disable(); - cpu = task_cpu(p); - if ((cpu != smp_processor_id()) && task_curr(p)) - smp_send_reschedule(cpu); - preempt_enable(); -} -EXPORT_SYMBOL_GPL(kick_process); -#endif /* CONFIG_SMP */ - -#ifdef CONFIG_SMP -/* - * ->cpus_allowed is protected by both rq->lock and p->pi_lock - */ -static int select_fallback_rq(int cpu, struct task_struct *p) -{ - int nid = cpu_to_node(cpu); - const struct cpumask *nodemask = NULL; - enum { cpuset, possible, fail } state = cpuset; - int dest_cpu; - - /* - * If the node that the cpu is on has been offlined, cpu_to_node() - * will return -1. There is no cpu on the node, and we should - * select the cpu on the other node. - */ - if (nid != -1) { - nodemask = cpumask_of_node(nid); - - /* Look for allowed, online CPU in same node. */ - for_each_cpu(dest_cpu, nodemask) { - if (!cpu_online(dest_cpu)) - continue; - if (!cpu_active(dest_cpu)) - continue; - if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) - return dest_cpu; - } - } - - for (;;) { - /* Any allowed, online CPU? */ - for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) { - if (!cpu_online(dest_cpu)) - continue; - if (!cpu_active(dest_cpu)) - continue; - goto out; - } - - switch (state) { - case cpuset: - /* No more Mr. Nice Guy. */ - cpuset_cpus_allowed_fallback(p); - state = possible; - break; - - case possible: - do_set_cpus_allowed(p, cpu_possible_mask); - state = fail; - break; - - case fail: - BUG(); - break; - } - } - -out: - if (state != cpuset) { - /* - * Don't tell them about moving exiting tasks or - * kernel threads (both mm NULL), since they never - * leave kernel. - */ - if (p->mm && printk_ratelimit()) { - printk_deferred("process %d (%s) no longer affine to cpu%d\n", - task_pid_nr(p), p->comm, cpu); - } - } - - return dest_cpu; -} - -/* - * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable. - */ -static inline -int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags) -{ - int cpu = p->sched_class->select_task_rq(p, sd_flags, wake_flags); - - /* - * In order not to call set_task_cpu() on a blocking task we need - * to rely on ttwu() to place the task on a valid ->cpus_allowed - * cpu. - * - * Since this is common to all placement strategies, this lives here. - * - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ - if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) || - !cpu_online(cpu))) - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -} - -static void update_avg(u64 *avg, u64 sample) -{ - s64 diff = sample - *avg; - *avg += diff >> 3; -} -#endif - -static void -ttwu_stat(struct task_struct *p, int cpu, int wake_flags) -{ -#ifdef CONFIG_SCHEDSTATS - struct rq *rq = this_rq(); - -#ifdef CONFIG_SMP - int this_cpu = smp_processor_id(); - - if (cpu == this_cpu) { - schedstat_inc(rq, ttwu_local); - schedstat_inc(p, se.statistics.nr_wakeups_local); - } else { - struct sched_domain *sd; - - schedstat_inc(p, se.statistics.nr_wakeups_remote); - rcu_read_lock(); - for_each_domain(this_cpu, sd) { - if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { - schedstat_inc(sd, ttwu_wake_remote); - break; - } - } - rcu_read_unlock(); - } - - if (wake_flags & WF_MIGRATED) - schedstat_inc(p, se.statistics.nr_wakeups_migrate); - -#endif /* CONFIG_SMP */ - - schedstat_inc(rq, ttwu_count); - schedstat_inc(p, se.statistics.nr_wakeups); - - if (wake_flags & WF_SYNC) - schedstat_inc(p, se.statistics.nr_wakeups_sync); - -#endif /* CONFIG_SCHEDSTATS */ -} - -static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags) -{ - activate_task(rq, p, en_flags); - p->on_rq = 1; - - /* if a worker is waking up, notify workqueue */ - if (p->flags & PF_WQ_WORKER) - wq_worker_waking_up(p, cpu_of(rq)); -} - -/* - * Mark the task runnable and perform wakeup-preemption. - */ -static void -ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) -{ - check_preempt_curr(rq, p, wake_flags); - trace_sched_wakeup(p, true); - - p->state = TASK_RUNNING; -#ifdef CONFIG_SMP - if (p->sched_class->task_woken) - p->sched_class->task_woken(rq, p); - - if (rq->idle_stamp) { - u64 delta = rq->clock - rq->idle_stamp; - u64 max = 2*sysctl_sched_migration_cost; - - if (delta > max) - rq->avg_idle = max; - else - update_avg(&rq->avg_idle, delta); - rq->idle_stamp = 0; - } -#endif -} - -static void -ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags) -{ -#ifdef CONFIG_SMP - if (p->sched_contributes_to_load) - rq->nr_uninterruptible--; -#endif - - ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING); - ttwu_do_wakeup(rq, p, wake_flags); -} - -/* - * Called in case the task @p isn't fully descheduled from its runqueue, - * in this case we must do a remote wakeup. Its a 'light' wakeup though, - * since all we need to do is flip p->state to TASK_RUNNING, since - * the task is still ->on_rq. - */ -static int ttwu_remote(struct task_struct *p, int wake_flags) -{ - struct rq *rq; - int ret = 0; - - rq = __task_rq_lock(p); - if (p->on_rq) { - ttwu_do_wakeup(rq, p, wake_flags); - ret = 1; - } - __task_rq_unlock(rq); - - return ret; -} - -#ifdef CONFIG_SMP -static void sched_ttwu_pending(void) -{ - struct rq *rq = this_rq(); - struct llist_node *llist = llist_del_all(&rq->wake_list); - struct task_struct *p; - - raw_spin_lock(&rq->lock); - - while (llist) { - p = llist_entry(llist, struct task_struct, wake_entry); - llist = llist_next(llist); - ttwu_do_activate(rq, p, 0); - } - - raw_spin_unlock(&rq->lock); -} - -void scheduler_ipi(void) -{ - int cpu = smp_processor_id(); - - if (llist_empty(&this_rq()->wake_list) - && !tick_nohz_full_cpu(cpu) - && !got_nohz_idle_kick() - && !got_boost_kick()) - return; - - if (got_boost_kick()) { - struct rq *rq = cpu_rq(cpu); - - if (rq->curr->sched_class == &fair_sched_class) - check_for_migration(rq, rq->curr); - clear_boost_kick(cpu); - } - - - /* - * Not all reschedule IPI handlers call irq_enter/irq_exit, since - * traditionally all their work was done from the interrupt return - * path. Now that we actually do some work, we need to make sure - * we do call them. - * - * Some archs already do call them, luckily irq_enter/exit nest - * properly. - * - * Arguably we should visit all archs and update all handlers, - * however a fair share of IPIs are still resched only so this would - * somewhat pessimize the simple resched case. - */ - irq_enter(); - tick_nohz_full_check(); - sched_ttwu_pending(); - - /* - * Check if someone kicked us for doing the nohz idle load balance. - */ - if (unlikely(got_nohz_idle_kick())) { - this_rq()->idle_balance = 1; - raise_softirq_irqoff(SCHED_SOFTIRQ); - } - irq_exit(); -} - -static void ttwu_queue_remote(struct task_struct *p, int cpu) -{ - if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) - smp_send_reschedule(cpu); -} - -bool cpus_share_cache(int this_cpu, int that_cpu) -{ - return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); -} -#endif /* CONFIG_SMP */ - -static void ttwu_queue(struct task_struct *p, int cpu) -{ - struct rq *rq = cpu_rq(cpu); - -#if defined(CONFIG_SMP) - if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { - sched_clock_cpu(cpu); /* sync clocks x-cpu */ - ttwu_queue_remote(p, cpu); - return; - } -#endif - - raw_spin_lock(&rq->lock); - ttwu_do_activate(rq, p, 0); - raw_spin_unlock(&rq->lock); -} - -__read_mostly unsigned int sysctl_sched_wakeup_load_threshold = 110; - -/** - * try_to_wake_up - wake up a thread - * @p: the thread to be awakened - * @state: the mask of task states that can be woken - * @wake_flags: wake modifier flags (WF_*) - * - * Put it on the run-queue if it's not already there. The "current" - * thread is always on the run-queue (except when the actual - * re-schedule is in progress), and as such you're allowed to do - * the simpler "current->state = TASK_RUNNING" to mark yourself - * runnable without the overhead of this. - * - * Returns %true if @p was woken up, %false if it was already running - * or @state didn't match @p's state. - */ -static int -try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) -{ - unsigned long flags; - int cpu, success = 0; - unsigned long src_cpu; - int notify = 0; - struct migration_notify_data mnd; - int heavy_task = 0; -#ifdef CONFIG_SMP - struct rq *rq; - u64 wallclock; -#endif - - /* - * If we are going to wake up a thread waiting for CONDITION we - * need to ensure that CONDITION=1 done by the caller can not be - * reordered with p->state check below. This pairs with mb() in - * set_current_state() the waiting thread does. - */ - smp_mb__before_spinlock(); - raw_spin_lock_irqsave(&p->pi_lock, flags); - src_cpu = cpu = task_cpu(p); - - if (!(p->state & state)) - goto out; - - success = 1; /* we're going to change ->state */ - - /* - * Ensure we load p->on_rq _after_ p->state, otherwise it would - * be possible to, falsely, observe p->on_rq == 0 and get stuck - * in smp_cond_load_acquire() below. - * - * sched_ttwu_pending() try_to_wake_up() - * [S] p->on_rq = 1; [L] P->state - * UNLOCK rq->lock -----. - * \ - * +--- RMB - * schedule() / - * LOCK rq->lock -----' - * UNLOCK rq->lock - * - * [task p] - * [S] p->state = UNINTERRUPTIBLE [L] p->on_rq - * - * Pairs with the UNLOCK+LOCK on rq->lock from the - * last wakeup of our task and the schedule that got our task - * current. - */ - smp_rmb(); - if (p->on_rq && ttwu_remote(p, wake_flags)) - goto stat; - -#ifdef CONFIG_SMP - /* - * If the owning (remote) cpu is still in the middle of schedule() with - * this task as prev, wait until its done referencing the task. - */ - while (p->on_cpu) - cpu_relax(); - /* - * Pairs with the smp_wmb() in finish_lock_switch(). - */ - smp_rmb(); - - rq = cpu_rq(task_cpu(p)); - - raw_spin_lock(&rq->lock); - wallclock = sched_clock(); - update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); - heavy_task = heavy_task_wakeup(p, rq, TASK_WAKE); - update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); - raw_spin_unlock(&rq->lock); - - p->sched_contributes_to_load = !!task_contributes_to_load(p); - p->state = TASK_WAKING; - - if (p->sched_class->task_waking) - p->sched_class->task_waking(p); - - cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags); - - /* Refresh src_cpu as it could have changed since we last read it */ - src_cpu = task_cpu(p); - if (src_cpu != cpu) { - wake_flags |= WF_MIGRATED; - set_task_cpu(p, cpu); - } else { - note_run_start(p, wallclock); - } -#endif /* CONFIG_SMP */ - - ttwu_queue(p, cpu); -stat: - ttwu_stat(p, cpu, wake_flags); - - if (task_notify_on_migrate(p)) { - mnd.src_cpu = src_cpu; - mnd.dest_cpu = cpu; - mnd.load = pct_task_load(p); - - /* - * Call the migration notifier with mnd for foreground task - * migrations as well as for wakeups if their load is above - * sysctl_sched_wakeup_load_threshold. This would prompt the - * cpu-boost to boost the CPU frequency on wake up of a heavy - * weight foreground task - */ - if ((src_cpu != cpu) || (mnd.load > - sysctl_sched_wakeup_load_threshold)) - notify = 1; - } - -out: - raw_spin_unlock_irqrestore(&p->pi_lock, flags); - - if (notify) - atomic_notifier_call_chain(&migration_notifier_head, - 0, (void *)&mnd); - - if (!same_freq_domain(src_cpu, cpu)) { - check_for_freq_change(cpu_rq(cpu)); - check_for_freq_change(cpu_rq(src_cpu)); - } else if (heavy_task) - check_for_freq_change(cpu_rq(cpu)); - - return success; -} - -/** - * try_to_wake_up_local - try to wake up a local task with rq lock held - * @p: the thread to be awakened - * - * Put @p on the run-queue if it's not already there. The caller must - * ensure that this_rq() is locked, @p is bound to this_rq() and not - * the current task. - */ -static void try_to_wake_up_local(struct task_struct *p) -{ - struct rq *rq = task_rq(p); - - if (rq != this_rq() || p == current) { - printk_deferred("%s: Failed to wakeup task %d (%s), rq = %p, this_rq = %p, p = %p, current = %p\n", - __func__, task_pid_nr(p), p->comm, rq, - this_rq(), p, current); - return; - } - - lockdep_assert_held(&rq->lock); - - if (!raw_spin_trylock(&p->pi_lock)) { - raw_spin_unlock(&rq->lock); - raw_spin_lock(&p->pi_lock); - raw_spin_lock(&rq->lock); - } - - if (!(p->state & TASK_NORMAL)) - goto out; - - if (!p->on_rq) { - u64 wallclock = sched_clock(); - - update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); - update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); - ttwu_activate(rq, p, ENQUEUE_WAKEUP); - } - - ttwu_do_wakeup(rq, p, 0); - ttwu_stat(p, smp_processor_id(), 0); -out: - raw_spin_unlock(&p->pi_lock); - /* Todo : Send cpufreq notifier */ -} - -/** - * wake_up_process - Wake up a specific process - * @p: The process to be woken up. - * - * Attempt to wake up the nominated process and move it to the set of runnable - * processes. Returns 1 if the process was woken up, 0 if it was already - * running. - * - * It may be assumed that this function implies a write memory barrier before - * changing the task state if and only if any tasks are woken up. - */ -int wake_up_process(struct task_struct *p) -{ - return try_to_wake_up(p, TASK_NORMAL, 0); -} -EXPORT_SYMBOL(wake_up_process); - -int wake_up_state(struct task_struct *p, unsigned int state) -{ - return try_to_wake_up(p, state, 0); -} - -/* - * Perform scheduler related setup for a newly forked process p. - * p is forked by current. - * - * __sched_fork() is basic setup used by init_idle() too: - */ -static void __sched_fork(struct task_struct *p) -{ - p->on_rq = 0; - - p->se.on_rq = 0; - p->se.exec_start = 0; - p->se.sum_exec_runtime = 0; - p->se.prev_sum_exec_runtime = 0; - p->se.nr_migrations = 0; - p->se.vruntime = 0; - - INIT_LIST_HEAD(&p->se.group_node); - -/* - * Load-tracking only depends on SMP, FAIR_GROUP_SCHED dependency below may be - * removed when useful for applications beyond shares distribution (e.g. - * load-balance). - */ -#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED) - init_new_task_load(p); -#endif -#ifdef CONFIG_SCHEDSTATS - memset(&p->se.statistics, 0, sizeof(p->se.statistics)); -#endif - - INIT_LIST_HEAD(&p->rt.run_list); - -#ifdef CONFIG_PREEMPT_NOTIFIERS - INIT_HLIST_HEAD(&p->preempt_notifiers); -#endif - -#ifdef CONFIG_NUMA_BALANCING - if (p->mm && atomic_read(&p->mm->mm_users) == 1) { - p->mm->numa_next_scan = jiffies; - p->mm->numa_next_reset = jiffies; - p->mm->numa_scan_seq = 0; - } - - p->node_stamp = 0ULL; - p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0; - p->numa_migrate_seq = p->mm ? p->mm->numa_scan_seq - 1 : 0; - p->numa_scan_period = sysctl_numa_balancing_scan_delay; - p->numa_work.next = &p->numa_work; -#endif /* CONFIG_NUMA_BALANCING */ -} - -#ifdef CONFIG_NUMA_BALANCING -#ifdef CONFIG_SCHED_DEBUG -void set_numabalancing_state(bool enabled) -{ - if (enabled) - sched_feat_set("NUMA"); - else - sched_feat_set("NO_NUMA"); -} -#else -__read_mostly bool numabalancing_enabled; - -void set_numabalancing_state(bool enabled) -{ - numabalancing_enabled = enabled; -} -#endif /* CONFIG_SCHED_DEBUG */ -#endif /* CONFIG_NUMA_BALANCING */ - -/* - * fork()/clone()-time setup: - */ -void sched_fork(unsigned long flags, struct task_struct *p) -{ - //unsigned long flags; - int cpu = get_cpu(); - - __sched_fork(p); - /* - * We mark the process as running here. This guarantees that - * nobody will actually run it, and a signal or other external - * event cannot wake it up and insert it on the runqueue either. - */ - p->state = TASK_RUNNING; - - /* - * Make sure we do not leak PI boosting priority to the child. - */ - p->prio = current->normal_prio; - - /* - * Revert to default priority/policy on fork if requested. - */ - if (unlikely(p->sched_reset_on_fork)) { - if (task_has_rt_policy(p)) { - p->policy = SCHED_NORMAL; - p->static_prio = NICE_TO_PRIO(0); - p->rt_priority = 0; - } else if (PRIO_TO_NICE(p->static_prio) < 0) - p->static_prio = NICE_TO_PRIO(0); - - p->prio = p->normal_prio = __normal_prio(p); - set_load_weight(p); - - /* - * We don't need the reset flag anymore after the fork. It has - * fulfilled its duty: - */ - p->sched_reset_on_fork = 0; - } - - if (!rt_prio(p->prio)) - p->sched_class = &fair_sched_class; - - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - - /* - * The child is not yet in the pid-hash so no cgroup attach races, - * and the cgroup is pinned to this child due to cgroup_fork() - * is ran before sched_fork(). - * - * Silence PROVE_RCU. - */ - raw_spin_lock_irqsave(&p->pi_lock, flags); - set_task_cpu(p, cpu); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); - -#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) - if (likely(sched_info_on())) - memset(&p->sched_info, 0, sizeof(p->sched_info)); -#endif -#if defined(CONFIG_SMP) - p->on_cpu = 0; -#endif -#ifdef CONFIG_PREEMPT_COUNT - /* Want to start with kernel preemption disabled. */ - task_thread_info(p)->preempt_count = 1; -#endif -#ifdef CONFIG_SMP - plist_node_init(&p->pushable_tasks, MAX_PRIO); -#endif - - put_cpu(); -} - -/* - * wake_up_new_task - wake up a newly created task for the first time. - * - * This function will do some initial scheduler statistics housekeeping - * that must be done for every newly created context, then puts the task - * on the runqueue and wakes it. - */ -void wake_up_new_task(struct task_struct *p) -{ - unsigned long flags; - struct rq *rq; - - raw_spin_lock_irqsave(&p->pi_lock, flags); -#ifdef CONFIG_SMP - /* - * Fork balancing, do it here and not earlier because: - * - cpus_allowed can change in the fork path - * - any previously selected cpu might disappear through hotplug - */ - set_task_cpu(p, select_task_rq(p, SD_BALANCE_FORK, 0)); -#endif - - rq = __task_rq_lock(p); - mark_task_starting(p); - activate_task(rq, p, 0); - p->on_rq = 1; - trace_sched_wakeup_new(p, true); - check_preempt_curr(rq, p, WF_FORK); -#ifdef CONFIG_SMP - if (p->sched_class->task_woken) - p->sched_class->task_woken(rq, p); -#endif - task_rq_unlock(rq, p, &flags); -} - -#ifdef CONFIG_PREEMPT_NOTIFIERS - -/** - * preempt_notifier_register - tell me when current is being preempted & rescheduled - * @notifier: notifier struct to register - */ -void preempt_notifier_register(struct preempt_notifier *notifier) -{ - hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); -} -EXPORT_SYMBOL_GPL(preempt_notifier_register); - -/** - * preempt_notifier_unregister - no longer interested in preemption notifications - * @notifier: notifier struct to unregister - * - * This is safe to call from within a preemption notifier. - */ -void preempt_notifier_unregister(struct preempt_notifier *notifier) -{ - hlist_del(¬ifier->link); -} -EXPORT_SYMBOL_GPL(preempt_notifier_unregister); - -static void fire_sched_in_preempt_notifiers(struct task_struct *curr) -{ - struct preempt_notifier *notifier; - - hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) - notifier->ops->sched_in(notifier, raw_smp_processor_id()); -} - -static void -fire_sched_out_preempt_notifiers(struct task_struct *curr, - struct task_struct *next) -{ - struct preempt_notifier *notifier; - - hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) - notifier->ops->sched_out(notifier, next); -} - -#else /* !CONFIG_PREEMPT_NOTIFIERS */ - -static void fire_sched_in_preempt_notifiers(struct task_struct *curr) -{ -} - -static void -fire_sched_out_preempt_notifiers(struct task_struct *curr, - struct task_struct *next) -{ -} - -#endif /* CONFIG_PREEMPT_NOTIFIERS */ - -/** - * prepare_task_switch - prepare to switch tasks - * @rq: the runqueue preparing to switch - * @prev: the current task that is being switched out - * @next: the task we are going to switch to. - * - * This is called with the rq lock held and interrupts off. It must - * be paired with a subsequent finish_task_switch after the context - * switch. - * - * prepare_task_switch sets up locking and calls architecture specific - * hooks. - */ -static inline void -prepare_task_switch(struct rq *rq, struct task_struct *prev, - struct task_struct *next) -{ - trace_sched_switch(prev, next); - sched_info_switch(prev, next); - perf_event_task_sched_out(prev, next); - fire_sched_out_preempt_notifiers(prev, next); - prepare_lock_switch(rq, next); - prepare_arch_switch(next); -} - -/** - * finish_task_switch - clean up after a task-switch - * @rq: runqueue associated with task-switch - * @prev: the thread we just switched away from. - * - * finish_task_switch must be called after the context switch, paired - * with a prepare_task_switch call before the context switch. - * finish_task_switch will reconcile locking set up by prepare_task_switch, - * and do any other architecture-specific cleanup actions. - * - * Note that we may have delayed dropping an mm in context_switch(). If - * so, we finish that here outside of the runqueue lock. (Doing it - * with the lock held can cause deadlocks; see schedule() for - * details.) - */ -static void finish_task_switch(struct rq *rq, struct task_struct *prev) - __releases(rq->lock) -{ - struct mm_struct *mm = rq->prev_mm; - long prev_state; - - rq->prev_mm = NULL; - - /* - * A task struct has one reference for the use as "current". - * If a task dies, then it sets TASK_DEAD in tsk->state and calls - * schedule one last time. The schedule call will never return, and - * the scheduled task must drop that reference. - * The test for TASK_DEAD must occur while the runqueue locks are - * still held, otherwise prev could be scheduled on another cpu, die - * there before we look at prev->state, and then the reference would - * be dropped twice. - * Manfred Spraul - */ - prev_state = prev->state; - vtime_task_switch(prev); - finish_arch_switch(prev); - perf_event_task_sched_in(prev, current); - finish_lock_switch(rq, prev); - finish_arch_post_lock_switch(); - - fire_sched_in_preempt_notifiers(current); - if (mm) - mmdrop(mm); - if (unlikely(prev_state == TASK_DEAD)) { - /* - * Remove function-return probe instances associated with this - * task and put them back on the free list. - */ - kprobe_flush_task(prev); - put_task_struct(prev); - } - - tick_nohz_task_switch(current); -} - -#ifdef CONFIG_SMP - -/* assumes rq->lock is held */ -static inline void pre_schedule(struct rq *rq, struct task_struct *prev) -{ - if (prev->sched_class->pre_schedule) - prev->sched_class->pre_schedule(rq, prev); -} - -/* rq->lock is NOT held, but preemption is disabled */ -static inline void post_schedule(struct rq *rq) -{ - if (rq->post_schedule) { - unsigned long flags; - - raw_spin_lock_irqsave(&rq->lock, flags); - if (rq->curr->sched_class->post_schedule) - rq->curr->sched_class->post_schedule(rq); - raw_spin_unlock_irqrestore(&rq->lock, flags); - - rq->post_schedule = 0; - } -} - -#else - -static inline void pre_schedule(struct rq *rq, struct task_struct *p) -{ -} - -static inline void post_schedule(struct rq *rq) -{ -} - -#endif - -/** - * schedule_tail - first thing a freshly forked thread must call. - * @prev: the thread we just switched away from. - */ -asmlinkage void schedule_tail(struct task_struct *prev) - __releases(rq->lock) -{ - struct rq *rq = this_rq(); - - finish_task_switch(rq, prev); - - /* - * FIXME: do we need to worry about rq being invalidated by the - * task_switch? - */ - post_schedule(rq); - -#ifdef __ARCH_WANT_UNLOCKED_CTXSW - /* In this case, finish_task_switch does not reenable preemption */ - preempt_enable(); -#endif - if (current->set_child_tid) - put_user(task_pid_vnr(current), current->set_child_tid); -} - -/* - * context_switch - switch to the new MM and the new - * thread's register state. - */ -static inline void -context_switch(struct rq *rq, struct task_struct *prev, - struct task_struct *next) -{ - struct mm_struct *mm, *oldmm; - - prepare_task_switch(rq, prev, next); - - mm = next->mm; - oldmm = prev->active_mm; - /* - * For paravirt, this is coupled with an exit in switch_to to - * combine the page table reload and the switch backend into - * one hypercall. - */ - arch_start_context_switch(prev); - - if (!mm) { - next->active_mm = oldmm; - atomic_inc(&oldmm->mm_count); - enter_lazy_tlb(oldmm, next); - } else - switch_mm(oldmm, mm, next); - - if (!prev->mm) { - prev->active_mm = NULL; - rq->prev_mm = oldmm; - } - /* - * Since the runqueue lock will be released by the next - * task (which is an invalid locking op but in the case - * of the scheduler it's an obvious special-case), so we - * do an early lockdep release here: - */ -#ifndef __ARCH_WANT_UNLOCKED_CTXSW - spin_release(&rq->lock.dep_map, 1, _THIS_IP_); -#endif - - context_tracking_task_switch(prev, next); - /* Here we just switch the register state and the stack. */ - switch_to(prev, next, prev); - - barrier(); - /* - * this_rq must be evaluated again because prev may have moved - * CPUs since it called schedule(), thus the 'rq' on its stack - * frame will be invalid. - */ - finish_task_switch(this_rq(), prev); -} - -/* - * nr_running and nr_context_switches: - * - * externally visible scheduler statistics: current number of runnable - * threads, total number of context switches performed since bootup. - */ -unsigned long nr_running(void) -{ - unsigned long i, sum = 0; - - for_each_online_cpu(i) - sum += cpu_rq(i)->nr_running; - - return sum; -} - -unsigned long long nr_context_switches(void) -{ - int i; - unsigned long long sum = 0; - - for_each_possible_cpu(i) - sum += cpu_rq(i)->nr_switches; - - return sum; -} - -unsigned long nr_iowait(void) -{ - unsigned long i, sum = 0; - - for_each_possible_cpu(i) - sum += atomic_read(&cpu_rq(i)->nr_iowait); - - return sum; -} - -unsigned long nr_iowait_cpu(int cpu) -{ - struct rq *this = cpu_rq(cpu); - return atomic_read(&this->nr_iowait); -} - -unsigned long this_cpu_load(void) -{ - struct rq *this = this_rq(); - return this->cpu_load[0]; -} - - -/* - * Global load-average calculations - * - * We take a distributed and async approach to calculating the global load-avg - * in order to minimize overhead. - * - * The global load average is an exponentially decaying average of nr_running + - * nr_uninterruptible. - * - * Once every LOAD_FREQ: - * - * nr_active = 0; - * for_each_possible_cpu(cpu) - * nr_active += cpu_of(cpu)->nr_running + cpu_of(cpu)->nr_uninterruptible; - * - * avenrun[n] = avenrun[0] * exp_n + nr_active * (1 - exp_n) - * - * Due to a number of reasons the above turns in the mess below: - * - * - for_each_possible_cpu() is prohibitively expensive on machines with - * serious number of cpus, therefore we need to take a distributed approach - * to calculating nr_active. - * - * \Sum_i x_i(t) = \Sum_i x_i(t) - x_i(t_0) | x_i(t_0) := 0 - * = \Sum_i { \Sum_j=1 x_i(t_j) - x_i(t_j-1) } - * - * So assuming nr_active := 0 when we start out -- true per definition, we - * can simply take per-cpu deltas and fold those into a global accumulate - * to obtain the same result. See calc_load_fold_active(). - * - * Furthermore, in order to avoid synchronizing all per-cpu delta folding - * across the machine, we assume 10 ticks is sufficient time for every - * cpu to have completed this task. - * - * This places an upper-bound on the IRQ-off latency of the machine. Then - * again, being late doesn't loose the delta, just wrecks the sample. - * - * - cpu_rq()->nr_uninterruptible isn't accurately tracked per-cpu because - * this would add another cross-cpu cacheline miss and atomic operation - * to the wakeup path. Instead we increment on whatever cpu the task ran - * when it went into uninterruptible state and decrement on whatever cpu - * did the wakeup. This means that only the sum of nr_uninterruptible over - * all cpus yields the correct result. - * - * This covers the NO_HZ=n code, for extra head-aches, see the comment below. - */ - -/* Variables and functions for calc_load */ -static atomic_long_t calc_load_tasks; -static unsigned long calc_load_update; -unsigned long avenrun[3]; -EXPORT_SYMBOL(avenrun); /* should be removed */ - -/** - * get_avenrun - get the load average array - * @loads: pointer to dest load array - * @offset: offset to add - * @shift: shift count to shift the result left - * - * These values are estimates at best, so no need for locking. - */ -void get_avenrun(unsigned long *loads, unsigned long offset, int shift) -{ - loads[0] = (avenrun[0] + offset) << shift; - loads[1] = (avenrun[1] + offset) << shift; - loads[2] = (avenrun[2] + offset) << shift; -} - -static long calc_load_fold_active(struct rq *this_rq) -{ - long nr_active, delta = 0; - - nr_active = this_rq->nr_running; - nr_active += (long) this_rq->nr_uninterruptible; - - if (nr_active != this_rq->calc_load_active) { - delta = nr_active - this_rq->calc_load_active; - this_rq->calc_load_active = nr_active; - } - - return delta; -} - -/* - * a1 = a0 * e + a * (1 - e) - */ -static unsigned long -calc_load(unsigned long load, unsigned long exp, unsigned long active) -{ - unsigned long newload; - - newload = load * exp + active * (FIXED_1 - exp); - if (active >= load) - newload += FIXED_1-1; - - return newload / FIXED_1; -} - -#ifdef CONFIG_NO_HZ_COMMON -/* - * Handle NO_HZ for the global load-average. - * - * Since the above described distributed algorithm to compute the global - * load-average relies on per-cpu sampling from the tick, it is affected by - * NO_HZ. - * - * The basic idea is to fold the nr_active delta into a global idle-delta upon - * entering NO_HZ state such that we can include this as an 'extra' cpu delta - * when we read the global state. - * - * Obviously reality has to ruin such a delightfully simple scheme: - * - * - When we go NO_HZ idle during the window, we can negate our sample - * contribution, causing under-accounting. - * - * We avoid this by keeping two idle-delta counters and flipping them - * when the window starts, thus separating old and new NO_HZ load. - * - * The only trick is the slight shift in index flip for read vs write. - * - * 0s 5s 10s 15s - * +10 +10 +10 +10 - * |-|-----------|-|-----------|-|-----------|-| - * r:0 0 1 1 0 0 1 1 0 - * w:0 1 1 0 0 1 1 0 0 - * - * This ensures we'll fold the old idle contribution in this window while - * accumlating the new one. - * - * - When we wake up from NO_HZ idle during the window, we push up our - * contribution, since we effectively move our sample point to a known - * busy state. - * - * This is solved by pushing the window forward, and thus skipping the - * sample, for this cpu (effectively using the idle-delta for this cpu which - * was in effect at the time the window opened). This also solves the issue - * of having to deal with a cpu having been in NOHZ idle for multiple - * LOAD_FREQ intervals. - * - * When making the ILB scale, we should try to pull this in as well. - */ -static atomic_long_t calc_load_idle[2]; -static int calc_load_idx; - -static inline int calc_load_write_idx(void) -{ - int idx = calc_load_idx; - - /* - * See calc_global_nohz(), if we observe the new index, we also - * need to observe the new update time. - */ - smp_rmb(); - - /* - * If the folding window started, make sure we start writing in the - * next idle-delta. - */ - if (!time_before(jiffies, calc_load_update)) - idx++; - - return idx & 1; -} - -static inline int calc_load_read_idx(void) -{ - return calc_load_idx & 1; -} - -void calc_load_enter_idle(void) -{ - struct rq *this_rq = this_rq(); - long delta; - - /* - * We're going into NOHZ mode, if there's any pending delta, fold it - * into the pending idle delta. - */ - delta = calc_load_fold_active(this_rq); - if (delta) { - int idx = calc_load_write_idx(); - atomic_long_add(delta, &calc_load_idle[idx]); - } -} - -void calc_load_exit_idle(void) -{ - struct rq *this_rq = this_rq(); - - /* - * If we're still before the sample window, we're done. - */ - if (time_before(jiffies, this_rq->calc_load_update)) - return; - - /* - * We woke inside or after the sample window, this means we're already - * accounted through the nohz accounting, so skip the entire deal and - * sync up for the next window. - */ - this_rq->calc_load_update = calc_load_update; - if (time_before(jiffies, this_rq->calc_load_update + 10)) - this_rq->calc_load_update += LOAD_FREQ; -} - -static long calc_load_fold_idle(void) -{ - int idx = calc_load_read_idx(); - long delta = 0; - - if (atomic_long_read(&calc_load_idle[idx])) - delta = atomic_long_xchg(&calc_load_idle[idx], 0); - - return delta; -} - -/** - * fixed_power_int - compute: x^n, in O(log n) time - * - * @x: base of the power - * @frac_bits: fractional bits of @x - * @n: power to raise @x to. - * - * By exploiting the relation between the definition of the natural power - * function: x^n := x*x*...*x (x multiplied by itself for n times), and - * the binary encoding of numbers used by computers: n := \Sum n_i * 2^i, - * (where: n_i \elem {0, 1}, the binary vector representing n), - * we find: x^n := x^(\Sum n_i * 2^i) := \Prod x^(n_i * 2^i), which is - * of course trivially computable in O(log_2 n), the length of our binary - * vector. - */ -static unsigned long -fixed_power_int(unsigned long x, unsigned int frac_bits, unsigned int n) -{ - unsigned long result = 1UL << frac_bits; - - if (n) for (;;) { - if (n & 1) { - result *= x; - result += 1UL << (frac_bits - 1); - result >>= frac_bits; - } - n >>= 1; - if (!n) - break; - x *= x; - x += 1UL << (frac_bits - 1); - x >>= frac_bits; - } - - return result; -} - -/* - * a1 = a0 * e + a * (1 - e) - * - * a2 = a1 * e + a * (1 - e) - * = (a0 * e + a * (1 - e)) * e + a * (1 - e) - * = a0 * e^2 + a * (1 - e) * (1 + e) - * - * a3 = a2 * e + a * (1 - e) - * = (a0 * e^2 + a * (1 - e) * (1 + e)) * e + a * (1 - e) - * = a0 * e^3 + a * (1 - e) * (1 + e + e^2) - * - * ... - * - * an = a0 * e^n + a * (1 - e) * (1 + e + ... + e^n-1) [1] - * = a0 * e^n + a * (1 - e) * (1 - e^n)/(1 - e) - * = a0 * e^n + a * (1 - e^n) - * - * [1] application of the geometric series: - * - * n 1 - x^(n+1) - * S_n := \Sum x^i = ------------- - * i=0 1 - x - */ -static unsigned long -calc_load_n(unsigned long load, unsigned long exp, - unsigned long active, unsigned int n) -{ - - return calc_load(load, fixed_power_int(exp, FSHIFT, n), active); -} - -/* - * NO_HZ can leave us missing all per-cpu ticks calling - * calc_load_account_active(), but since an idle CPU folds its delta into - * calc_load_tasks_idle per calc_load_account_idle(), all we need to do is fold - * in the pending idle delta if our idle period crossed a load cycle boundary. - * - * Once we've updated the global active value, we need to apply the exponential - * weights adjusted to the number of cycles missed. - */ -static void calc_global_nohz(void) -{ - long delta, active, n; - - if (!time_before(jiffies, calc_load_update + 10)) { - /* - * Catch-up, fold however many we are behind still - */ - delta = jiffies - calc_load_update - 10; - n = 1 + (delta / LOAD_FREQ); - - active = atomic_long_read(&calc_load_tasks); - active = active > 0 ? active * FIXED_1 : 0; - - avenrun[0] = calc_load_n(avenrun[0], EXP_1, active, n); - avenrun[1] = calc_load_n(avenrun[1], EXP_5, active, n); - avenrun[2] = calc_load_n(avenrun[2], EXP_15, active, n); - - calc_load_update += n * LOAD_FREQ; - } - - /* - * Flip the idle index... - * - * Make sure we first write the new time then flip the index, so that - * calc_load_write_idx() will see the new time when it reads the new - * index, this avoids a double flip messing things up. - */ - smp_wmb(); - calc_load_idx++; -} -#else /* !CONFIG_NO_HZ_COMMON */ - -static inline long calc_load_fold_idle(void) { return 0; } -static inline void calc_global_nohz(void) { } - -#endif /* CONFIG_NO_HZ_COMMON */ - -/* - * calc_load - update the avenrun load estimates 10 ticks after the - * CPUs have updated calc_load_tasks. - */ -void calc_global_load(unsigned long ticks) -{ - long active, delta; - - if (time_before(jiffies, calc_load_update + 10)) - return; - - /* - * Fold the 'old' idle-delta to include all NO_HZ cpus. - */ - delta = calc_load_fold_idle(); - if (delta) - atomic_long_add(delta, &calc_load_tasks); - - active = atomic_long_read(&calc_load_tasks); - active = active > 0 ? active * FIXED_1 : 0; - - avenrun[0] = calc_load(avenrun[0], EXP_1, active); - avenrun[1] = calc_load(avenrun[1], EXP_5, active); - avenrun[2] = calc_load(avenrun[2], EXP_15, active); - - calc_load_update += LOAD_FREQ; - - /* - * In case we idled for multiple LOAD_FREQ intervals, catch up in bulk. - */ - calc_global_nohz(); -} - -/* - * Called from update_cpu_load() to periodically update this CPU's - * active count. - */ -static void calc_load_account_active(struct rq *this_rq) -{ - long delta; - - if (time_before(jiffies, this_rq->calc_load_update)) - return; - - delta = calc_load_fold_active(this_rq); - if (delta) - atomic_long_add(delta, &calc_load_tasks); - - this_rq->calc_load_update += LOAD_FREQ; -} - -/* - * End of global load-average stuff - */ - -/* - * The exact cpuload at various idx values, calculated at every tick would be - * load = (2^idx - 1) / 2^idx * load + 1 / 2^idx * cur_load - * - * If a cpu misses updates for n-1 ticks (as it was idle) and update gets called - * on nth tick when cpu may be busy, then we have: - * load = ((2^idx - 1) / 2^idx)^(n-1) * load - * load = (2^idx - 1) / 2^idx) * load + 1 / 2^idx * cur_load - * - * decay_load_missed() below does efficient calculation of - * load = ((2^idx - 1) / 2^idx)^(n-1) * load - * avoiding 0..n-1 loop doing load = ((2^idx - 1) / 2^idx) * load - * - * The calculation is approximated on a 128 point scale. - * degrade_zero_ticks is the number of ticks after which load at any - * particular idx is approximated to be zero. - * degrade_factor is a precomputed table, a row for each load idx. - * Each column corresponds to degradation factor for a power of two ticks, - * based on 128 point scale. - * Example: - * row 2, col 3 (=12) says that the degradation at load idx 2 after - * 8 ticks is 12/128 (which is an approximation of exact factor 3^8/4^8). - * - * With this power of 2 load factors, we can degrade the load n times - * by looking at 1 bits in n and doing as many mult/shift instead of - * n mult/shifts needed by the exact degradation. - */ -#define DEGRADE_SHIFT 7 -static const unsigned char - degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128}; -static const unsigned char - degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = { - {0, 0, 0, 0, 0, 0, 0, 0}, - {64, 32, 8, 0, 0, 0, 0, 0}, - {96, 72, 40, 12, 1, 0, 0}, - {112, 98, 75, 43, 15, 1, 0}, - {120, 112, 98, 76, 45, 16, 2} }; - -/* - * Update cpu_load for any missed ticks, due to tickless idle. The backlog - * would be when CPU is idle and so we just decay the old load without - * adding any new load. - */ -static unsigned long -decay_load_missed(unsigned long load, unsigned long missed_updates, int idx) -{ - int j = 0; - - if (!missed_updates) - return load; - - if (missed_updates >= degrade_zero_ticks[idx]) - return 0; - - if (idx == 1) - return load >> missed_updates; - - while (missed_updates) { - if (missed_updates % 2) - load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT; - - missed_updates >>= 1; - j++; - } - return load; -} - -/* - * Update rq->cpu_load[] statistics. This function is usually called every - * scheduler tick (TICK_NSEC). With tickless idle this will not be called - * every tick. We fix it up based on jiffies. - */ -static void __update_cpu_load(struct rq *this_rq, unsigned long this_load, - unsigned long pending_updates) -{ - int i, scale; - - this_rq->nr_load_updates++; - - /* Update our load: */ - this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */ - for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) { - unsigned long old_load, new_load; - - /* scale is effectively 1 << i now, and >> i divides by scale */ - - old_load = this_rq->cpu_load[i]; - old_load = decay_load_missed(old_load, pending_updates - 1, i); - new_load = this_load; - /* - * Round up the averaging division if load is increasing. This - * prevents us from getting stuck on 9 if the load is 10, for - * example. - */ - if (new_load > old_load) - new_load += scale - 1; - - this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i; - } - - sched_avg_update(this_rq); -} - -#ifdef CONFIG_NO_HZ_COMMON -/* - * There is no sane way to deal with nohz on smp when using jiffies because the - * cpu doing the jiffies update might drift wrt the cpu doing the jiffy reading - * causing off-by-one errors in observed deltas; {0,2} instead of {1,1}. - * - * Therefore we cannot use the delta approach from the regular tick since that - * would seriously skew the load calculation. However we'll make do for those - * updates happening while idle (nohz_idle_balance) or coming out of idle - * (tick_nohz_idle_exit). - * - * This means we might still be one tick off for nohz periods. - */ - -/* - * Called from nohz_idle_balance() to update the load ratings before doing the - * idle balance. - */ -void update_idle_cpu_load(struct rq *this_rq) -{ - unsigned long curr_jiffies = ACCESS_ONCE(jiffies); - unsigned long load = this_rq->load.weight; - unsigned long pending_updates; - - /* - * bail if there's load or we're actually up-to-date. - */ - if (load || curr_jiffies == this_rq->last_load_update_tick) - return; - - pending_updates = curr_jiffies - this_rq->last_load_update_tick; - this_rq->last_load_update_tick = curr_jiffies; - - __update_cpu_load(this_rq, load, pending_updates); -} - -/* - * Called from tick_nohz_idle_exit() -- try and fix up the ticks we missed. - */ -void update_cpu_load_nohz(void) -{ - struct rq *this_rq = this_rq(); - unsigned long curr_jiffies = ACCESS_ONCE(jiffies); - unsigned long pending_updates; - - if (curr_jiffies == this_rq->last_load_update_tick) - return; - - raw_spin_lock(&this_rq->lock); - pending_updates = curr_jiffies - this_rq->last_load_update_tick; - if (pending_updates) { - this_rq->last_load_update_tick = curr_jiffies; - /* - * We were idle, this means load 0, the current load might be - * !0 due to remote wakeups and the sort. - */ - __update_cpu_load(this_rq, 0, pending_updates); - } - raw_spin_unlock(&this_rq->lock); -} -#endif /* CONFIG_NO_HZ_COMMON */ - -/* - * Called from scheduler_tick() - */ -static void update_cpu_load_active(struct rq *this_rq) -{ - /* - * See the mess around update_idle_cpu_load() / update_cpu_load_nohz(). - */ - this_rq->last_load_update_tick = jiffies; - __update_cpu_load(this_rq, this_rq->load.weight, 1); - - calc_load_account_active(this_rq); -} - -#if defined(CONFIG_SMP) - -/* - * sched_exec - execve() is a valuable balancing opportunity, because at - * this point the task has the smallest effective memory and cache footprint. - */ -void sched_exec(void) -{ - struct task_struct *p = current; - unsigned long flags; - int dest_cpu; - - if (sched_enable_hmp) - return; - - raw_spin_lock_irqsave(&p->pi_lock, flags); - dest_cpu = p->sched_class->select_task_rq(p, SD_BALANCE_EXEC, 0); - if (dest_cpu == smp_processor_id()) - goto unlock; - - if (likely(cpu_active(dest_cpu))) { - struct migration_arg arg = { p, dest_cpu }; - - raw_spin_unlock_irqrestore(&p->pi_lock, flags); - stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); - return; - } -unlock: - raw_spin_unlock_irqrestore(&p->pi_lock, flags); -} - -#endif - -DEFINE_PER_CPU(struct kernel_stat, kstat); -DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); - -EXPORT_PER_CPU_SYMBOL(kstat); -EXPORT_PER_CPU_SYMBOL(kernel_cpustat); - -/* - * Return any ns on the sched_clock that have not yet been accounted in - * @p in case that task is currently running. - * - * Called with task_rq_lock() held on @rq. - */ -static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq) -{ - u64 ns = 0; - - if (task_current(rq, p)) { - update_rq_clock(rq); - ns = rq->clock_task - p->se.exec_start; - if ((s64)ns < 0) - ns = 0; - } - - return ns; -} - -unsigned long long task_delta_exec(struct task_struct *p) -{ - unsigned long flags; - struct rq *rq; - u64 ns = 0; - - rq = task_rq_lock(p, &flags); - ns = do_task_delta_exec(p, rq); - task_rq_unlock(rq, p, &flags); - - return ns; -} - -/* - * Return accounted runtime for the task. - * In case the task is currently running, return the runtime plus current's - * pending runtime that have not been accounted yet. - */ -unsigned long long task_sched_runtime(struct task_struct *p) -{ - unsigned long flags; - struct rq *rq; - u64 ns = 0; - - rq = task_rq_lock(p, &flags); - ns = p->se.sum_exec_runtime + do_task_delta_exec(p, rq); - task_rq_unlock(rq, p, &flags); - - return ns; -} - -/* - * This function gets called by the timer code, with HZ frequency. - * We call it with interrupts disabled. - */ -void scheduler_tick(void) -{ - int cpu = smp_processor_id(); - struct rq *rq = cpu_rq(cpu); - struct task_struct *curr = rq->curr; - - sched_clock_tick(); - - raw_spin_lock(&rq->lock); - set_window_start(rq); - update_rq_clock(rq); - update_cpu_load_active(rq); - curr->sched_class->task_tick(rq, curr, 0); - update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), 0); - raw_spin_unlock(&rq->lock); - - perf_event_task_tick(); - -#ifdef CONFIG_SMP - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq, cpu); -#endif - rq_last_tick_reset(rq); - if (curr->sched_class == &fair_sched_class) - check_for_migration(rq, curr); -} - -#ifdef CONFIG_NO_HZ_FULL -/** - * scheduler_tick_max_deferment - * - * Keep at least one tick per second when a single - * active task is running because the scheduler doesn't - * yet completely support full dynticks environment. - * - * This makes sure that uptime, CFS vruntime, load - * balancing, etc... continue to move forward, even - * with a very low granularity. - */ -u64 scheduler_tick_max_deferment(void) -{ - struct rq *rq = this_rq(); - unsigned long next, now = ACCESS_ONCE(jiffies); - - next = rq->last_sched_tick + HZ; - - if (time_before_eq(next, now)) - return 0; - - return jiffies_to_usecs(next - now) * NSEC_PER_USEC; -} -#endif - -notrace unsigned long get_parent_ip(unsigned long addr) -{ - if (in_lock_functions(addr)) { - addr = CALLER_ADDR2; - if (in_lock_functions(addr)) - addr = CALLER_ADDR3; - } - return addr; -} - -#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \ - defined(CONFIG_PREEMPT_TRACER)) - -void __kprobes add_preempt_count(int val) -{ -#ifdef CONFIG_DEBUG_PREEMPT - /* - * Underflow? - */ - if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) - return; -#endif - preempt_count() += val; -#ifdef CONFIG_DEBUG_PREEMPT - /* - * Spinlock count overflowing soon? - */ - DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= - PREEMPT_MASK - 10); -#endif - if (preempt_count() == val) - trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); -} -EXPORT_SYMBOL(add_preempt_count); - -void __kprobes sub_preempt_count(int val) -{ -#ifdef CONFIG_DEBUG_PREEMPT - /* - * Underflow? - */ - if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) - return; - /* - * Is the spinlock portion underflowing? - */ - if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && - !(preempt_count() & PREEMPT_MASK))) - return; -#endif - - if (preempt_count() == val) - trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); - preempt_count() -= val; -} -EXPORT_SYMBOL(sub_preempt_count); - -#endif - -/* - * Print scheduling while atomic bug: - */ -static noinline void __schedule_bug(struct task_struct *prev) -{ - if (oops_in_progress) - return; - - printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", - prev->comm, prev->pid, preempt_count()); - - debug_show_held_locks(prev); - print_modules(); - if (irqs_disabled()) - print_irqtrace_events(prev); - dump_stack(); - add_taint(TAINT_WARN, LOCKDEP_STILL_OK); -} - -/* - * Various schedule()-time debugging checks and statistics: - */ -static inline void schedule_debug(struct task_struct *prev) -{ - /* - * Test if we are atomic. Since do_exit() needs to call into - * schedule() atomically, we ignore that path for now. - * Otherwise, whine if we are scheduling when we should not be. - */ - if (unlikely(in_atomic_preempt_off() && !prev->exit_state)) - __schedule_bug(prev); - rcu_sleep_check(); - - profile_hit(SCHED_PROFILING, __builtin_return_address(0)); - - schedstat_inc(this_rq(), sched_count); -} - -static void put_prev_task(struct rq *rq, struct task_struct *prev) -{ - if (prev->on_rq || rq->skip_clock_update < 0) - update_rq_clock(rq); - prev->sched_class->put_prev_task(rq, prev); -} - -/* - * Pick up the highest-prio task: - */ -static inline struct task_struct * -pick_next_task(struct rq *rq) -{ - const struct sched_class *class; - struct task_struct *p; - - /* - * Optimization: we know that if all tasks are in - * the fair class we can call that function directly: - */ - if (likely(rq->nr_running == rq->cfs.h_nr_running)) { - p = fair_sched_class.pick_next_task(rq); - if (likely(p)) - return p; - } - - for_each_class(class) { - p = class->pick_next_task(rq); - if (p) - return p; - } - - BUG(); /* the idle class will always have a runnable task */ -} - -/* - * __schedule() is the main scheduler function. - * - * The main means of driving the scheduler and thus entering this function are: - * - * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. - * - * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return - * paths. For example, see arch/x86/entry_64.S. - * - * To drive preemption between tasks, the scheduler sets the flag in timer - * interrupt handler scheduler_tick(). - * - * 3. Wakeups don't really cause entry into schedule(). They add a - * task to the run-queue and that's it. - * - * Now, if the new task added to the run-queue preempts the current - * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets - * called on the nearest possible occasion: - * - * - If the kernel is preemptible (CONFIG_PREEMPT=y): - * - * - in syscall or exception context, at the next outmost - * preempt_enable(). (this might be as soon as the wake_up()'s - * spin_unlock()!) - * - * - in IRQ context, return from interrupt-handler to - * preemptible context - * - * - If the kernel is not preemptible (CONFIG_PREEMPT is not set) - * then at the next: - * - * - cond_resched() call - * - explicit schedule() call - * - return from syscall or exception to user-space - * - return from interrupt-handler to user-space - */ -static void __sched __schedule(void) -{ - struct task_struct *prev, *next; - unsigned long *switch_count; - struct rq *rq; - int cpu; - u64 wallclock; - -need_resched: - preempt_disable(); - cpu = smp_processor_id(); - rq = cpu_rq(cpu); - rcu_note_context_switch(cpu); - prev = rq->curr; - - schedule_debug(prev); - - if (sched_feat(HRTICK)) - hrtick_clear(rq); + context_tracking_task_switch(prev, next); + /* Here we just switch the register state and the stack. */ + switch_to(prev, next, prev); + barrier(); /* - * Make sure that signal_pending_state()->signal_pending() below - * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) - * done by the caller to avoid the race with signal_wake_up(). + * this_rq must be evaluated again because prev may have moved + * CPUs since it called schedule(), thus the 'rq' on its stack + * frame will be invalid. */ - smp_mb__before_spinlock(); - raw_spin_lock_irq(&rq->lock); - - switch_count = &prev->nivcsw; - if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { - if (unlikely(signal_pending_state(prev->state, prev))) { - prev->state = TASK_RUNNING; - } else { - deactivate_task(rq, prev, DEQUEUE_SLEEP); - prev->on_rq = 0; - - /* - * If a worker went to sleep, notify and ask workqueue - * whether it wants to wake up a task to maintain - * concurrency. - */ - if (prev->flags & PF_WQ_WORKER) { - struct task_struct *to_wakeup; + finish_task_switch(this_rq(), prev); +} - to_wakeup = wq_worker_sleeping(prev, cpu); - if (to_wakeup) - try_to_wake_up_local(to_wakeup); - } - } - switch_count = &prev->nvcsw; - } +/* + * nr_running and nr_context_switches: + * + * externally visible scheduler statistics: current number of runnable + * threads, total number of context switches performed since bootup. + */ +unsigned long nr_running(void) +{ + unsigned long i, sum = 0; - pre_schedule(rq, prev); + for_each_online_cpu(i) + sum += cpu_rq(i)->nr_running; - if (unlikely(!rq->nr_running)) - idle_balance(cpu, rq); + return sum; +} - put_prev_task(rq, prev); - next = pick_next_task(rq); - wallclock = sched_clock(); - update_task_ravg(prev, rq, PUT_PREV_TASK, wallclock, 0); - update_task_ravg(next, rq, PICK_NEXT_TASK, wallclock, 0); - clear_tsk_need_resched(prev); - rq->skip_clock_update = 0; +/* + * Check if only the current task is running on the cpu. + */ +bool single_task_running(void) +{ + if (cpu_rq(smp_processor_id())->nr_running == 1) + return true; + else + return false; +} +EXPORT_SYMBOL(single_task_running); + +unsigned long long nr_context_switches(void) +{ + int i; + unsigned long long sum = 0; - BUG_ON(task_cpu(next) != cpu_of(rq)); + for_each_possible_cpu(i) + sum += cpu_rq(i)->nr_switches; - if (likely(prev != next)) { - rq->nr_switches++; - rq->curr = next; - ++*switch_count; + return sum; +} - context_switch(rq, prev, next); /* unlocks the rq */ - /* - * The context switch have flipped the stack from under us - * and restored the local variables which were saved when - * this task called schedule() in the past. prev == current - * is still correct, but it can be moved to another cpu/rq. - */ - cpu = smp_processor_id(); - rq = cpu_rq(cpu); - } else - raw_spin_unlock_irq(&rq->lock); +unsigned long nr_iowait(void) +{ + unsigned long i, sum = 0; - post_schedule(rq); + for_each_possible_cpu(i) + sum += atomic_read(&cpu_rq(i)->nr_iowait); - sched_preempt_enable_no_resched(); - if (need_resched()) - goto need_resched; + return sum; } -static inline void sched_submit_work(struct task_struct *tsk) +unsigned long nr_iowait_cpu(int cpu) { - if (!tsk->state || tsk_is_pi_blocked(tsk)) - return; - /* - * If we are going to sleep and we have plugged IO queued, - * make sure to submit it to avoid deadlocks. - */ - if (blk_needs_flush_plug(tsk)) - blk_schedule_flush_plug(tsk); + struct rq *this = cpu_rq(cpu); + return atomic_read(&this->nr_iowait); } -asmlinkage void __sched schedule(void) +void get_iowait_load(unsigned long *nr_waiters, unsigned long *load) { - struct task_struct *tsk = current; - - sched_submit_work(tsk); - __schedule(); + struct rq *this = this_rq(); + *nr_waiters = atomic_read(&this->nr_iowait); + *load = this->cpu_load[0]; } -EXPORT_SYMBOL(schedule); -#ifdef CONFIG_CONTEXT_TRACKING -asmlinkage void __sched schedule_user(void) +#ifdef CONFIG_SMP + +/* + * sched_exec - execve() is a valuable balancing opportunity, because at + * this point the task has the smallest effective memory and cache footprint. + */ +void sched_exec(void) { - /* - * If we come here after a random call to set_need_resched(), - * or we have been woken up remotely but the IPI has not yet arrived, - * we haven't yet exited the RCU idle mode. Do it here manually until - * we find a better solution. - */ - user_exit(); - schedule(); - user_enter(); + struct task_struct *p = current; + unsigned long flags; + int dest_cpu; + + raw_spin_lock_irqsave(&p->pi_lock, flags); + dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0); + if (dest_cpu == smp_processor_id()) + goto unlock; + + if (likely(cpu_active(dest_cpu))) { + struct migration_arg arg = { p, dest_cpu }; + + raw_spin_unlock_irqrestore(&p->pi_lock, flags); + stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); + return; + } +unlock: + raw_spin_unlock_irqrestore(&p->pi_lock, flags); } + #endif -/** - * schedule_preempt_disabled - called with preemption disabled +DEFINE_PER_CPU(struct kernel_stat, kstat); +DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); + +EXPORT_PER_CPU_SYMBOL(kstat); +EXPORT_PER_CPU_SYMBOL(kernel_cpustat); + +/* + * Return any ns on the sched_clock that have not yet been accounted in + * @p in case that task is currently running. * - * Returns with preemption disabled. Note: preempt_count must be 1 + * Called with task_rq_lock() held on @rq. */ -void __sched schedule_preempt_disabled(void) +static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq) { - sched_preempt_enable_no_resched(); - schedule(); - preempt_disable(); + u64 ns = 0; + + if (task_current(rq, p)) { + update_rq_clock(rq); + ns = rq->clock_task - p->se.exec_start; + if ((s64)ns < 0) + ns = 0; + } + + return ns; +} + +unsigned long long task_delta_exec(struct task_struct *p) +{ + unsigned long flags; + struct rq *rq; + u64 ns = 0; + + rq = task_rq_lock(p, &flags); + ns = do_task_delta_exec(p, rq); + task_rq_unlock(rq, p, &flags); + + return ns; } -#ifdef CONFIG_PREEMPT /* - * this is the entry point to schedule() from in-kernel preemption - * off of preempt_enable. Kernel preemptions off return from interrupt - * occur there and call schedule directly. + * Return accounted runtime for the task. + * In case the task is currently running, return the runtime plus current's + * pending runtime that have not been accounted yet. */ -asmlinkage void __sched notrace preempt_schedule(void) + +unsigned long long task_sched_runtime(struct task_struct *p) { - struct thread_info *ti = current_thread_info(); + unsigned long flags; + struct rq *rq; + u64 ns; +#if defined(CONFIG_64BIT) && defined(CONFIG_SMP) /* - * If there is a non-zero preempt_count or interrupts are disabled, - * we do not want to preempt the current task. Just return.. + * 64-bit doesn't need locks to atomically read a 64bit value. + * So we have a optimization chance when the task's delta_exec is 0. + * Reading ->on_cpu is racy, but this is ok. + * + * If we race with it leaving cpu, we'll take a lock. So we're correct. + * If we race with it entering cpu, unaccounted time is 0. This is + * indistinguishable from the read occurring a few cycles earlier. + * If we see ->on_cpu without ->on_rq, the task is leaving, and has + * been accounted, so we're correct here as well. */ - if (likely(ti->preempt_count || irqs_disabled())) - return; + if (!p->on_cpu || !task_on_rq_queued(p)) + return p->se.sum_exec_runtime; +#endif - do { - add_preempt_count_notrace(PREEMPT_ACTIVE); - __schedule(); - sub_preempt_count_notrace(PREEMPT_ACTIVE); + rq = task_rq_lock(p, &flags); + /* + * Must be ->curr _and_ ->on_rq. If dequeued, we would + * project cycles that may never be accounted to this + * thread, breaking clock_gettime(). + */ + if (task_current(rq, p) && task_on_rq_queued(p)) { + update_rq_clock(rq); + p->sched_class->update_curr(rq); + } + ns = p->se.sum_exec_runtime; + task_rq_unlock(rq, p, &flags); - /* - * Check again in case we missed a preemption opportunity - * between schedule and now. - */ - barrier(); - } while (need_resched()); + return ns; } -EXPORT_SYMBOL(preempt_schedule); /* - * this is the entry point to schedule() from kernel preemption - * off of irq context. - * Note, that this is called and return with irqs disabled. This will - * protect us against recursive calling from irq. + * This function gets called by the timer code, with HZ frequency. + * We call it with interrupts disabled. */ -asmlinkage void __sched preempt_schedule_irq(void) +void scheduler_tick(void) { - struct thread_info *ti = current_thread_info(); - enum ctx_state prev_state; + int cpu = smp_processor_id(); + struct rq *rq = cpu_rq(cpu); + struct task_struct *curr = rq->curr; - /* Catch callers which need to be fixed */ - BUG_ON(ti->preempt_count || !irqs_disabled()); + sched_clock_tick(); - prev_state = exception_enter(); + raw_spin_lock(&rq->lock); + update_rq_clock(rq); + curr->sched_class->task_tick(rq, curr, 0); + update_cpu_load_active(rq); + raw_spin_unlock(&rq->lock); - do { - add_preempt_count(PREEMPT_ACTIVE); - local_irq_enable(); - __schedule(); - local_irq_disable(); - sub_preempt_count(PREEMPT_ACTIVE); + perf_event_task_tick(); +#ifdef CONFIG_MT_SCHED_MONITOR + if (smp_processor_id() == 0) /* only record by CPU#0 */ + mt_save_irq_counts(); +#endif +#ifdef CONFIG_SMP + rq->idle_balance = idle_cpu(cpu); + trigger_load_balance(rq); +#endif + rq_last_tick_reset(rq); +} - /* - * Check again in case we missed a preemption opportunity - * between schedule and now. - */ - barrier(); - } while (need_resched()); +#ifdef CONFIG_NO_HZ_FULL +/** + * scheduler_tick_max_deferment + * + * Keep at least one tick per second when a single + * active task is running because the scheduler doesn't + * yet completely support full dynticks environment. + * + * This makes sure that uptime, CFS vruntime, load + * balancing, etc... continue to move forward, even + * with a very low granularity. + * + * Return: Maximum deferment in nanoseconds. + */ +u64 scheduler_tick_max_deferment(void) +{ + struct rq *rq = this_rq(); + unsigned long next, now = ACCESS_ONCE(jiffies); - exception_exit(prev_state); -} + next = rq->last_sched_tick + HZ; -#endif /* CONFIG_PREEMPT */ + if (time_before_eq(next, now)) + return 0; -int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags, - void *key) -{ - return try_to_wake_up(curr->private, mode, wake_flags); + return jiffies_to_nsecs(next - now); } -EXPORT_SYMBOL(default_wake_function); +#endif -/* - * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just - * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve - * number) then we wake all the non-exclusive tasks and one exclusive task. - * - * There are circumstances in which we can try to wake a task which has already - * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns - * zero in this (rare) case, and we handle it by continuing to scan the queue. - */ -static void __wake_up_common(wait_queue_head_t *q, unsigned int mode, - int nr_exclusive, int wake_flags, void *key) +notrace unsigned long get_parent_ip(unsigned long addr) { - wait_queue_t *curr, *next; + if (in_lock_functions(addr)) { + addr = CALLER_ADDR2; + if (in_lock_functions(addr)) + addr = CALLER_ADDR3; + } + return addr; +} - list_for_each_entry_safe(curr, next, &q->task_list, task_list) { - unsigned flags = curr->flags; +#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \ + defined(CONFIG_PREEMPT_TRACER)) - if (curr->func(curr, mode, wake_flags, key) && - (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive) - break; +void preempt_count_add(int val) +{ +#ifdef CONFIG_DEBUG_PREEMPT + /* + * Underflow? + */ + if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) + return; +#endif + preempt_count() += val; +#ifdef CONFIG_DEBUG_PREEMPT + /* + * Spinlock count overflowing soon? + */ + DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= + PREEMPT_MASK - 10); +#endif + if (preempt_count() == val) { + unsigned long ip = get_parent_ip(CALLER_ADDR1); +#ifdef CONFIG_DEBUG_PREEMPT + current->preempt_disable_ip = ip; +#endif + trace_preempt_off(CALLER_ADDR0, ip); +#if defined(CONFIG_PREEMPT_MONITOR) && defined(CONFIG_MTPROF) + if (unlikely(__raw_get_cpu_var(mtsched_mon_enabled) & 0x1)) { + /* current->t_add_prmpt = sched_clock(); */ + MT_trace_preempt_off(); + } +#endif } } +EXPORT_SYMBOL(preempt_count_add); +#if 0 +// TJK +NOKPROBE_SYMBOL(preempt_count_add); -/** - * __wake_up - wake up threads blocked on a waitqueue. - * @q: the waitqueue - * @mode: which threads - * @nr_exclusive: how many wake-one or wake-many threads to wake up - * @key: is directly passed to the wakeup function - * - * It may be assumed that this function implies a write memory barrier before - * changing the task state if and only if any tasks are woken up. - */ -void __wake_up(wait_queue_head_t *q, unsigned int mode, - int nr_exclusive, void *key) +void preempt_count_sub(int val) { - unsigned long flags; +#ifdef CONFIG_DEBUG_PREEMPT + /* + * Underflow? + */ + if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) + return; + /* + * Is the spinlock portion underflowing? + */ + if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && + !(preempt_count() & PREEMPT_MASK))) + return; +#endif - spin_lock_irqsave(&q->lock, flags); - __wake_up_common(q, mode, nr_exclusive, 0, key); - spin_unlock_irqrestore(&q->lock, flags); + if (preempt_count() == val) + trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); +#if defined(CONFIG_PREEMPT_MONITOR) && defined(CONFIG_MTPROF) + if (unlikely(__raw_get_cpu_var(mtsched_mon_enabled) & 0x1)) + MT_trace_preempt_on(); +#endif + __preempt_count_sub(val); } -EXPORT_SYMBOL(__wake_up); +EXPORT_SYMBOL(preempt_count_sub); +NOKPROBE_SYMBOL(preempt_count_sub); +#endif + +#endif /* - * Same as __wake_up but called with the spinlock in wait_queue_head_t held. + * Print scheduling while atomic bug: */ -void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr) +static noinline void __schedule_bug(struct task_struct *prev) { - __wake_up_common(q, mode, nr, 0, NULL); -} -EXPORT_SYMBOL_GPL(__wake_up_locked); + if (oops_in_progress) + return; -void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key) -{ - __wake_up_common(q, mode, 1, 0, key); + printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", + prev->comm, prev->pid, preempt_count()); + + debug_show_held_locks(prev); + print_modules(); + if (irqs_disabled()) + print_irqtrace_events(prev); +#ifdef CONFIG_DEBUG_PREEMPT + if (in_atomic_preempt_off()) { + pr_err("Preemption disabled at:"); + print_ip_sym(current->preempt_disable_ip); + pr_cont("\n"); + } +#endif + dump_stack(); + add_taint(TAINT_WARN, LOCKDEP_STILL_OK); } -EXPORT_SYMBOL_GPL(__wake_up_locked_key); -/** - * __wake_up_sync_key - wake up threads blocked on a waitqueue. - * @q: the waitqueue - * @mode: which threads - * @nr_exclusive: how many wake-one or wake-many threads to wake up - * @key: opaque value to be passed to wakeup targets - * - * The sync wakeup differs that the waker knows that it will schedule - * away soon, so while the target thread will be woken up, it will not - * be migrated to another CPU - ie. the two threads are 'synchronized' - * with each other. This can prevent needless bouncing between CPUs. - * - * On UP it can prevent extra preemption. - * - * It may be assumed that this function implies a write memory barrier before - * changing the task state if and only if any tasks are woken up. +/* + * Various schedule()-time debugging checks and statistics: */ -void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, - int nr_exclusive, void *key) +static inline void schedule_debug(struct task_struct *prev) { - unsigned long flags; - int wake_flags = WF_SYNC; - - if (unlikely(!q)) - return; +#ifdef CONFIG_SCHED_STACK_END_CHECK + BUG_ON(unlikely(task_stack_end_corrupted(prev))); +#endif + /* + * Test if we are atomic. Since do_exit() needs to call into + * schedule() atomically, we ignore that path. Otherwise whine + * if we are scheduling when we should not. + */ + if (unlikely(in_atomic_preempt_off() && prev->state != TASK_DEAD)) + __schedule_bug(prev); + rcu_sleep_check(); - if (unlikely(!nr_exclusive)) - wake_flags = 0; + profile_hit(SCHED_PROFILING, __builtin_return_address(0)); - spin_lock_irqsave(&q->lock, flags); - __wake_up_common(q, mode, nr_exclusive, wake_flags, key); - spin_unlock_irqrestore(&q->lock, flags); + schedstat_inc(this_rq(), sched_count); } -EXPORT_SYMBOL_GPL(__wake_up_sync_key); /* - * __wake_up_sync - see __wake_up_sync_key() + * Pick up the highest-prio task: */ -void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive) +static inline struct task_struct * +pick_next_task(struct rq *rq, struct task_struct *prev) { - __wake_up_sync_key(q, mode, nr_exclusive, NULL); + const struct sched_class *class = &fair_sched_class; + struct task_struct *p; + + /* + * Optimization: we know that if all tasks are in + * the fair class we can call that function directly: + */ + if (likely(prev->sched_class == class && + rq->nr_running == rq->cfs.h_nr_running)) { + p = fair_sched_class.pick_next_task(rq, prev); + if (unlikely(p == RETRY_TASK)) + goto again; + + /* assumes fair_sched_class->next == idle_sched_class */ + if (unlikely(!p)) + p = idle_sched_class.pick_next_task(rq, prev); + + return p; + } + +again: + for_each_class(class) { + p = class->pick_next_task(rq, prev); + if (p) { + if (unlikely(p == RETRY_TASK)) + goto again; + return p; + } + } + + BUG(); /* the idle class will always have a runnable task */ } -EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ -/** - * complete: - signals a single thread waiting on this completion - * @x: holds the state of this particular completion +/* + * __schedule() is the main scheduler function. * - * This will wake up a single thread waiting on this completion. Threads will be - * awakened in the same order in which they were queued. + * The main means of driving the scheduler and thus entering this function are: * - * See also complete_all(), wait_for_completion() and related routines. + * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. * - * It may be assumed that this function implies a write memory barrier before - * changing the task state if and only if any tasks are woken up. - */ -void complete(struct completion *x) -{ - unsigned long flags; - - spin_lock_irqsave(&x->wait.lock, flags); - x->done++; - __wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL); - spin_unlock_irqrestore(&x->wait.lock, flags); -} -EXPORT_SYMBOL(complete); - -/** - * complete_all: - signals all threads waiting on this completion - * @x: holds the state of this particular completion + * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return + * paths. For example, see arch/x86/entry_64.S. * - * This will wake up all threads waiting on this particular completion event. + * To drive preemption between tasks, the scheduler sets the flag in timer + * interrupt handler scheduler_tick(). * - * It may be assumed that this function implies a write memory barrier before - * changing the task state if and only if any tasks are woken up. + * 3. Wakeups don't really cause entry into schedule(). They add a + * task to the run-queue and that's it. + * + * Now, if the new task added to the run-queue preempts the current + * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets + * called on the nearest possible occasion: + * + * - If the kernel is preemptible (CONFIG_PREEMPT=y): + * + * - in syscall or exception context, at the next outmost + * preempt_enable(). (this might be as soon as the wake_up()'s + * spin_unlock()!) + * + * - in IRQ context, return from interrupt-handler to + * preemptible context + * + * - If the kernel is not preemptible (CONFIG_PREEMPT is not set) + * then at the next: + * + * - cond_resched() call + * - explicit schedule() call + * - return from syscall or exception to user-space + * - return from interrupt-handler to user-space */ -void complete_all(struct completion *x) +static void __sched __schedule(void) { - unsigned long flags; + struct task_struct *prev, *next; + unsigned long *switch_count; + struct rq *rq; + int cpu; - spin_lock_irqsave(&x->wait.lock, flags); - x->done += UINT_MAX/2; - __wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL); - spin_unlock_irqrestore(&x->wait.lock, flags); -} -EXPORT_SYMBOL(complete_all); +need_resched: + preempt_disable(); + cpu = smp_processor_id(); + rq = cpu_rq(cpu); + rcu_note_context_switch(cpu); + prev = rq->curr; + + schedule_debug(prev); + + if (sched_feat(HRTICK)) + hrtick_clear(rq); +#if defined(CONFIG_MT_SCHED_MONITOR) && defined(CONFIG_MTPROF) + __raw_get_cpu_var(MT_trace_in_sched) = 1; +#endif + + /* + * Make sure that signal_pending_state()->signal_pending() below + * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) + * done by the caller to avoid the race with signal_wake_up(). + */ + smp_mb__before_spinlock(); + raw_spin_lock_irq(&rq->lock); + + switch_count = &prev->nivcsw; + if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { + if (unlikely(signal_pending_state(prev->state, prev))) { + prev->state = TASK_RUNNING; + } else { + deactivate_task(rq, prev, DEQUEUE_SLEEP); + prev->on_rq = 0; -static inline long __sched -do_wait_for_common(struct completion *x, - long (*action)(long), long timeout, int state) -{ - if (!x->done) { - DECLARE_WAITQUEUE(wait, current); + /* + * If a worker went to sleep, notify and ask workqueue + * whether it wants to wake up a task to maintain + * concurrency. + */ + if (prev->flags & PF_WQ_WORKER) { + struct task_struct *to_wakeup; - __add_wait_queue_tail_exclusive(&x->wait, &wait); - do { - if (signal_pending_state(state, current)) { - timeout = -ERESTARTSYS; - break; + to_wakeup = wq_worker_sleeping(prev, cpu); + if (to_wakeup) + try_to_wake_up_local(to_wakeup); } - __set_current_state(state); - spin_unlock_irq(&x->wait.lock); - timeout = action(timeout); - spin_lock_irq(&x->wait.lock); - } while (!x->done && timeout); - __remove_wait_queue(&x->wait, &wait); - if (!x->done) - return timeout; + } + switch_count = &prev->nvcsw; } - x->done--; - return timeout ?: 1; -} -static inline long __sched -__wait_for_common(struct completion *x, - long (*action)(long), long timeout, int state) -{ - might_sleep(); + if (task_on_rq_queued(prev) || rq->skip_clock_update < 0) + update_rq_clock(rq); - spin_lock_irq(&x->wait.lock); - timeout = do_wait_for_common(x, action, timeout, state); - spin_unlock_irq(&x->wait.lock); - return timeout; -} + next = pick_next_task(rq, prev); + clear_tsk_need_resched(prev); + //TJK clear_preempt_need_resched(); + rq->skip_clock_update = 0; -static long __sched -wait_for_common(struct completion *x, long timeout, int state) -{ - return __wait_for_common(x, schedule_timeout, timeout, state); -} + if (likely(prev != next)) { + rq->nr_switches++; + rq->curr = next; + ++*switch_count; -static long __sched -wait_for_common_io(struct completion *x, long timeout, int state) -{ - return __wait_for_common(x, io_schedule_timeout, timeout, state); -} + context_switch(rq, prev, next); /* unlocks the rq */ + /* + * The context switch have flipped the stack from under us + * and restored the local variables which were saved when + * this task called schedule() in the past. prev == current + * is still correct, but it can be moved to another cpu/rq. + */ + cpu = smp_processor_id(); + rq = cpu_rq(cpu); + } else + raw_spin_unlock_irq(&rq->lock); -/** - * wait_for_completion: - waits for completion of a task - * @x: holds the state of this particular completion - * - * This waits to be signaled for completion of a specific task. It is NOT - * interruptible and there is no timeout. - * - * See also similar routines (i.e. wait_for_completion_timeout()) with timeout - * and interrupt capability. Also see complete(). - */ -void __sched wait_for_completion(struct completion *x) -{ - wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); -} -EXPORT_SYMBOL(wait_for_completion); +#if defined(CONFIG_MT_SCHED_MONITOR) && defined(CONFIG_MTPROF) + __raw_get_cpu_var(MT_trace_in_sched) = 0; +#endif + post_schedule(rq); -/** - * wait_for_completion_timeout: - waits for completion of a task (w/timeout) - * @x: holds the state of this particular completion - * @timeout: timeout value in jiffies - * - * This waits for either a completion of a specific task to be signaled or for a - * specified timeout to expire. The timeout is in jiffies. It is not - * interruptible. - * - * The return value is 0 if timed out, and positive (at least 1, or number of - * jiffies left till timeout) if completed. - */ -unsigned long __sched -wait_for_completion_timeout(struct completion *x, unsigned long timeout) -{ - return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE); + sched_preempt_enable_no_resched(); + if (need_resched()) + goto need_resched; } -EXPORT_SYMBOL(wait_for_completion_timeout); -/** - * wait_for_completion_io: - waits for completion of a task - * @x: holds the state of this particular completion - * - * This waits to be signaled for completion of a specific task. It is NOT - * interruptible and there is no timeout. The caller is accounted as waiting - * for IO. - */ -void __sched wait_for_completion_io(struct completion *x) +static inline void sched_submit_work(struct task_struct *tsk) { - wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); + if (!tsk->state || tsk_is_pi_blocked(tsk)) + return; + /* + * If we are going to sleep and we have plugged IO queued, + * make sure to submit it to avoid deadlocks. + */ + if (blk_needs_flush_plug(tsk)) + blk_schedule_flush_plug(tsk); } -EXPORT_SYMBOL(wait_for_completion_io); -/** - * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout) - * @x: holds the state of this particular completion - * @timeout: timeout value in jiffies - * - * This waits for either a completion of a specific task to be signaled or for a - * specified timeout to expire. The timeout is in jiffies. It is not - * interruptible. The caller is accounted as waiting for IO. - * - * The return value is 0 if timed out, and positive (at least 1, or number of - * jiffies left till timeout) if completed. - */ -unsigned long __sched -wait_for_completion_io_timeout(struct completion *x, unsigned long timeout) +asmlinkage void __sched schedule(void) { - return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE); -} -EXPORT_SYMBOL(wait_for_completion_io_timeout); + struct task_struct *tsk = current; -/** - * wait_for_completion_interruptible: - waits for completion of a task (w/intr) - * @x: holds the state of this particular completion - * - * This waits for completion of a specific task to be signaled. It is - * interruptible. - * - * The return value is -ERESTARTSYS if interrupted, 0 if completed. - */ -int __sched wait_for_completion_interruptible(struct completion *x) -{ - long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE); - if (t == -ERESTARTSYS) - return t; - return 0; + sched_submit_work(tsk); + __schedule(); } -EXPORT_SYMBOL(wait_for_completion_interruptible); +EXPORT_SYMBOL(schedule); -/** - * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr)) - * @x: holds the state of this particular completion - * @timeout: timeout value in jiffies - * - * This waits for either a completion of a specific task to be signaled or for a - * specified timeout to expire. It is interruptible. The timeout is in jiffies. - * - * The return value is -ERESTARTSYS if interrupted, 0 if timed out, - * positive (at least 1, or number of jiffies left till timeout) if completed. - */ -long __sched -wait_for_completion_interruptible_timeout(struct completion *x, - unsigned long timeout) +#ifdef CONFIG_CONTEXT_TRACKING +asmlinkage void __sched schedule_user(void) { - return wait_for_common(x, timeout, TASK_INTERRUPTIBLE); + /* + * If we come here after a random call to set_need_resched(), + * or we have been woken up remotely but the IPI has not yet arrived, + * we haven't yet exited the RCU idle mode. Do it here manually until + * we find a better solution. + * + * NB: There are buggy callers of this function. Ideally we + * should warn if prev_state != IN_USER, but that will trigger + * too frequently to make sense yet. + */ + enum ctx_state prev_state = exception_enter(); + schedule(); + exception_exit(prev_state); } -EXPORT_SYMBOL(wait_for_completion_interruptible_timeout); +#endif /** - * wait_for_completion_killable: - waits for completion of a task (killable) - * @x: holds the state of this particular completion - * - * This waits to be signaled for completion of a specific task. It can be - * interrupted by a kill signal. + * schedule_preempt_disabled - called with preemption disabled * - * The return value is -ERESTARTSYS if interrupted, 0 if completed. + * Returns with preemption disabled. Note: preempt_count must be 1 */ -int __sched wait_for_completion_killable(struct completion *x) +void __sched schedule_preempt_disabled(void) { - long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE); - if (t == -ERESTARTSYS) - return t; - return 0; + sched_preempt_enable_no_resched(); + schedule(); + preempt_disable(); } -EXPORT_SYMBOL(wait_for_completion_killable); -/** - * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable)) - * @x: holds the state of this particular completion - * @timeout: timeout value in jiffies - * - * This waits for either a completion of a specific task to be - * signaled or for a specified timeout to expire. It can be - * interrupted by a kill signal. The timeout is in jiffies. - * - * The return value is -ERESTARTSYS if interrupted, 0 if timed out, - * positive (at least 1, or number of jiffies left till timeout) if completed. +#ifdef CONFIG_PREEMPT +/* + * this is the entry point to schedule() from in-kernel preemption + * off of preempt_enable. Kernel preemptions off return from interrupt + * occur there and call schedule directly. */ -long __sched -wait_for_completion_killable_timeout(struct completion *x, - unsigned long timeout) +asmlinkage void __sched notrace preempt_schedule(void) { - return wait_for_common(x, timeout, TASK_KILLABLE); -} -EXPORT_SYMBOL(wait_for_completion_killable_timeout); + struct thread_info *ti = current_thread_info(); -/** - * try_wait_for_completion - try to decrement a completion without blocking - * @x: completion structure - * - * Returns: 0 if a decrement cannot be done without blocking - * 1 if a decrement succeeded. - * - * If a completion is being used as a counting completion, - * attempt to decrement the counter without blocking. This - * enables us to avoid waiting if the resource the completion - * is protecting is not available. - */ -bool try_wait_for_completion(struct completion *x) -{ - unsigned long flags; - int ret = 1; + /* + * If there is a non-zero preempt_count or interrupts are disabled, + * we do not want to preempt the current task. Just return.. + */ + if (likely(ti->preempt_count || irqs_disabled())) + return; - spin_lock_irqsave(&x->wait.lock, flags); - if (!x->done) - ret = 0; - else - x->done--; - spin_unlock_irqrestore(&x->wait.lock, flags); - return ret; + do { + add_preempt_count_notrace(PREEMPT_ACTIVE); + __schedule(); + sub_preempt_count_notrace(PREEMPT_ACTIVE); + + /* + * Check again in case we missed a preemption opportunity + * between schedule and now. + */ + barrier(); + } while (need_resched()); } -EXPORT_SYMBOL(try_wait_for_completion); +//NOKPROBE_SYMBOL(preempt_schedule); +EXPORT_SYMBOL(preempt_schedule); +//#ifdef CONFIG_CONTEXT_TRACKING /** - * completion_done - Test to see if a completion has any waiters - * @x: completion structure + * preempt_schedule_context - preempt_schedule called by tracing * - * Returns: 0 if there are waiters (wait_for_completion() in progress) - * 1 if there are no waiters. + * The tracing infrastructure uses preempt_enable_notrace to prevent + * recursion and tracing preempt enabling caused by the tracing + * infrastructure itself. But as tracing can happen in areas coming + * from userspace or just about to enter userspace, a preempt enable + * can occur before user_exit() is called. This will cause the scheduler + * to be called when the system is still in usermode. * + * To prevent this, the preempt_enable_notrace will use this function + * instead of preempt_schedule() to exit user context if needed before + * calling the scheduler. */ -bool completion_done(struct completion *x) +asmlinkage void __sched notrace preempt_schedule_context(void) { - unsigned long flags; - int ret = 1; + enum ctx_state prev_ctx; - spin_lock_irqsave(&x->wait.lock, flags); - if (!x->done) - ret = 0; - spin_unlock_irqrestore(&x->wait.lock, flags); - return ret; -} -EXPORT_SYMBOL(completion_done); + if (likely(!preemptible())) + return; -static long __sched -sleep_on_common(wait_queue_head_t *q, int state, long timeout) -{ - unsigned long flags; - wait_queue_t wait; + do { + add_preempt_count(PREEMPT_ACTIVE); + /* + * Needs preempt disabled in case user_exit() is traced + * and the tracer calls preempt_enable_notrace() causing + * an infinite recursion. + */ + prev_ctx = exception_enter(); + __schedule(); + exception_exit(prev_ctx); - init_waitqueue_entry(&wait, current); + sub_preempt_count(PREEMPT_ACTIVE); + barrier(); + } while (need_resched()); +} +EXPORT_SYMBOL_GPL(preempt_schedule_context); +//#endif /* CONFIG_CONTEXT_TRACKING */ - __set_current_state(state); +#endif /* CONFIG_PREEMPT */ - spin_lock_irqsave(&q->lock, flags); - __add_wait_queue(q, &wait); - spin_unlock(&q->lock); - timeout = schedule_timeout(timeout); - spin_lock_irq(&q->lock); - __remove_wait_queue(q, &wait); - spin_unlock_irqrestore(&q->lock, flags); +/* + * this is the entry point to schedule() from kernel preemption + * off of irq context. + * Note, that this is called and return with irqs disabled. This will + * protect us against recursive calling from irq. + */ +asmlinkage void __sched preempt_schedule_irq(void) +{ + struct thread_info *ti = current_thread_info(); + enum ctx_state prev_state; - return timeout; -} + /* Catch callers which need to be fixed */ + BUG_ON(ti->preempt_count || !irqs_disabled()); -void __sched interruptible_sleep_on(wait_queue_head_t *q) -{ - sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); -} -EXPORT_SYMBOL(interruptible_sleep_on); + prev_state = exception_enter(); -long __sched -interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout) -{ - return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout); -} -EXPORT_SYMBOL(interruptible_sleep_on_timeout); + do { + add_preempt_count(PREEMPT_ACTIVE); + local_irq_enable(); + __schedule(); + local_irq_disable(); + sub_preempt_count(PREEMPT_ACTIVE); -void __sched sleep_on(wait_queue_head_t *q) -{ - sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); + /* + * Check again in case we missed a preemption opportunity + * between schedule and now. + */ + barrier(); + } while (need_resched()); + + exception_exit(prev_state); } -EXPORT_SYMBOL(sleep_on); -long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout) +int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags, + void *key) { - return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout); + return try_to_wake_up(curr->private, mode, wake_flags); } -EXPORT_SYMBOL(sleep_on_timeout); +EXPORT_SYMBOL(default_wake_function); #ifdef CONFIG_RT_MUTEXES @@ -5308,15 +3126,16 @@ EXPORT_SYMBOL(sleep_on_timeout); * This function changes the 'effective' priority of a task. It does * not touch ->normal_prio like __setscheduler(). * - * Used by the rt_mutex code to implement priority inheritance logic. + * Used by the rt_mutex code to implement priority inheritance + * logic. Call site only calls if the priority of the task changed. */ void rt_mutex_setprio(struct task_struct *p, int prio) { - int oldprio, on_rq, running; + int oldprio, queued, running, enqueue_flag = 0; struct rq *rq; const struct sched_class *prev_class; - BUG_ON(prio < 0 || prio > MAX_PRIO); + BUG_ON(prio > MAX_PRIO); rq = __task_rq_lock(p); @@ -5341,24 +3160,52 @@ void rt_mutex_setprio(struct task_struct *p, int prio) trace_sched_pi_setprio(p, prio); oldprio = p->prio; prev_class = p->sched_class; - on_rq = p->on_rq; + queued = task_on_rq_queued(p); running = task_current(rq, p); - if (on_rq) + if (queued) dequeue_task(rq, p, 0); if (running) - p->sched_class->put_prev_task(rq, p); + put_prev_task(rq, p); - if (rt_prio(prio)) + /* + * Boosting condition are: + * 1. -rt task is running and holds mutex A + * --> -dl task blocks on mutex A + * + * 2. -dl task is running and holds mutex A + * --> -dl task blocks on mutex A and could preempt the + * running task + */ + if (dl_prio(prio)) { + struct task_struct *pi_task = rt_mutex_get_top_task(p); + if (!dl_prio(p->normal_prio) || + (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) { + p->dl.dl_boosted = 1; + p->dl.dl_throttled = 0; + enqueue_flag = ENQUEUE_REPLENISH; + } else + p->dl.dl_boosted = 0; + p->sched_class = &dl_sched_class; + } else if (rt_prio(prio)) { + if (dl_prio(oldprio)) + p->dl.dl_boosted = 0; + if (oldprio < prio) + enqueue_flag = ENQUEUE_HEAD; p->sched_class = &rt_sched_class; - else + } else { + if (dl_prio(oldprio)) + p->dl.dl_boosted = 0; + if (rt_prio(oldprio)) + p->rt.timeout = 0; p->sched_class = &fair_sched_class; + } p->prio = prio; if (running) p->sched_class->set_curr_task(rq); - if (on_rq) - enqueue_task(rq, p, oldprio < prio ? ENQUEUE_HEAD : 0); + if (queued) + enqueue_task(rq, p, enqueue_flag); check_class_changed(rq, p, prev_class, oldprio); out_unlock: @@ -5368,11 +3215,11 @@ void rt_mutex_setprio(struct task_struct *p, int prio) void set_user_nice(struct task_struct *p, long nice) { - int old_prio, delta, on_rq; + int old_prio, delta, queued; unsigned long flags; struct rq *rq; - if (TASK_NICE(p) == nice || nice < -20 || nice > 19) + if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE) return; /* * We have to be careful, if called from sys_setpriority(), @@ -5383,14 +3230,14 @@ void set_user_nice(struct task_struct *p, long nice) * The RT priorities are set via sched_setscheduler(), but we still * allow the 'normal' nice value to be set - but as expected * it wont have any effect on scheduling until the task is - * SCHED_FIFO/SCHED_RR: + * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR: */ - if (task_has_rt_policy(p)) { + if (task_has_dl_policy(p) || task_has_rt_policy(p)) { p->static_prio = NICE_TO_PRIO(nice); goto out_unlock; } - on_rq = p->on_rq; - if (on_rq) + queued = task_on_rq_queued(p); + if (queued) dequeue_task(rq, p, 0); p->static_prio = NICE_TO_PRIO(nice); @@ -5399,14 +3246,14 @@ void set_user_nice(struct task_struct *p, long nice) p->prio = effective_prio(p); delta = p->prio - old_prio; - if (on_rq) { + if (queued) { enqueue_task(rq, p, 0); /* * If the task increased its priority or is running and * lowered its priority, then reschedule its CPU: */ if (delta < 0 || (delta > 0 && task_running(rq, p))) - resched_task(rq->curr); + resched_curr(rq); } out_unlock: task_rq_unlock(rq, p, &flags); @@ -5421,7 +3268,7 @@ EXPORT_SYMBOL(set_user_nice); int can_nice(const struct task_struct *p, const int nice) { /* convert nice value [19,-20] to rlimit style value [1,40] */ - int nice_rlim = 20 - nice; + int nice_rlim = nice_to_rlimit(nice); return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) || capable(CAP_SYS_NICE)); @@ -5445,17 +3292,10 @@ SYSCALL_DEFINE1(nice, int, increment) * We don't have to worry. Conceptually one call occurs first * and we have a single winner. */ - if (increment < -40) - increment = -40; - if (increment > 40) - increment = 40; - - nice = TASK_NICE(current) + increment; - if (nice < -20) - nice = -20; - if (nice > 19) - nice = 19; + increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH); + nice = task_nice(current) + increment; + nice = clamp_val(nice, MIN_NICE, MAX_NICE); if (increment < 0 && !can_nice(current, nice)) return -EPERM; @@ -5473,7 +3313,7 @@ SYSCALL_DEFINE1(nice, int, increment) * task_prio - return the priority value of a given task. * @p: the task in question. * - * This is the priority value as seen by users in /proc. + * Return: The priority value as seen by users in /proc. * RT tasks are offset by -200. Normal tasks are centered * around 0, value goes from -16 to +15. */ @@ -5482,19 +3322,11 @@ int task_prio(const struct task_struct *p) return p->prio - MAX_RT_PRIO; } -/** - * task_nice - return the nice value of a given task. - * @p: the task in question. - */ -int task_nice(const struct task_struct *p) -{ - return TASK_NICE(p); -} -EXPORT_SYMBOL(task_nice); - /** * idle_cpu - is a given cpu idle currently? * @cpu: the processor in question. + * + * Return: 1 if the CPU is currently idle. 0 otherwise. */ int idle_cpu(int cpu) { @@ -5517,6 +3349,8 @@ int idle_cpu(int cpu) /** * idle_task - return the idle task for a given cpu. * @cpu: the processor in question. + * + * Return: The idle task for the cpu @cpu. */ struct task_struct *idle_task(int cpu) { @@ -5526,24 +3360,57 @@ struct task_struct *idle_task(int cpu) /** * find_process_by_pid - find a process with a matching PID value. * @pid: the pid in question. + * + * The task of @pid, if found. %NULL otherwise. + */ +static struct task_struct *find_process_by_pid(pid_t pid) +{ + return pid ? find_task_by_vpid(pid) : current; +} + +/* + * This function initializes the sched_dl_entity of a newly becoming + * SCHED_DEADLINE task. + * + * Only the static values are considered here, the actual runtime and the + * absolute deadline will be properly calculated when the task is enqueued + * for the first time with its new policy. */ -static struct task_struct *find_process_by_pid(pid_t pid) +static void +__setparam_dl(struct task_struct *p, const struct sched_attr *attr) { - return pid ? find_task_by_vpid(pid) : current; + struct sched_dl_entity *dl_se = &p->dl; + + init_dl_task_timer(dl_se); + dl_se->dl_runtime = attr->sched_runtime; + dl_se->dl_deadline = attr->sched_deadline; + dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline; + dl_se->flags = attr->sched_flags; + dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); + dl_se->dl_throttled = 0; + dl_se->dl_new = 1; + dl_se->dl_yielded = 0; } -/* Actually do priority change: must hold pi & rq lock. */ -static void __setscheduler(struct rq *rq, struct task_struct *p, - const struct sched_attr *attr) +/* + * sched_setparam() passes in -1 for its policy, to let the functions + * it calls know not to change it. + */ +#define SETPARAM_POLICY -1 + +static void __setscheduler_params(struct task_struct *p, + const struct sched_attr *attr) { int policy = attr->sched_policy; - if (policy == -1) /* setparam */ + if (policy == SETPARAM_POLICY) policy = p->policy; p->policy = policy; - if (fair_policy(policy)) + if (dl_policy(policy)) + __setparam_dl(p, attr); + else if (fair_policy(policy)) p->static_prio = NICE_TO_PRIO(attr->sched_nice); /* @@ -5552,17 +3419,88 @@ static void __setscheduler(struct rq *rq, struct task_struct *p, * getparam()/getattr() don't report silly values for !rt tasks. */ p->rt_priority = attr->sched_priority; - p->normal_prio = normal_prio(p); - p->prio = rt_mutex_getprio(p); + set_load_weight(p); +} + +/* Actually do priority change: must hold pi & rq lock. */ +static void __setscheduler(struct rq *rq, struct task_struct *p, + const struct sched_attr *attr, bool keep_boost) +{ + __setscheduler_params(p, attr); - if (rt_prio(p->prio)) + /* + * Keep a potential priority boosting if called from + * sched_setscheduler(). + */ +#ifdef TJK_HMP + if (keep_boost) + p->prio = rt_mutex_get_effective_prio(p, normal_prio(p)); + else +#endif + p->prio = normal_prio(p); + + if (dl_prio(p->prio)) + p->sched_class = &dl_sched_class; + else if (rt_prio(p->prio)) p->sched_class = &rt_sched_class; else p->sched_class = &fair_sched_class; +} - set_load_weight(p); +static void +__getparam_dl(struct task_struct *p, struct sched_attr *attr) +{ + struct sched_dl_entity *dl_se = &p->dl; + + attr->sched_priority = p->rt_priority; + attr->sched_runtime = dl_se->dl_runtime; + attr->sched_deadline = dl_se->dl_deadline; + attr->sched_period = dl_se->dl_period; + attr->sched_flags = dl_se->flags; +} + +/* + * This function validates the new parameters of a -deadline task. + * We ask for the deadline not being zero, and greater or equal + * than the runtime, as well as the period of being zero or + * greater than deadline. Furthermore, we have to be sure that + * user parameters are above the internal resolution of 1us (we + * check sched_runtime only since it is always the smaller one) and + * below 2^63 ns (we have to check both sched_deadline and + * sched_period, as the latter can be zero). + */ +static bool +__checkparam_dl(const struct sched_attr *attr) +{ + /* deadline != 0 */ + if (attr->sched_deadline == 0) + return false; + + /* + * Since we truncate DL_SCALE bits, make sure we're at least + * that big. + */ + if (attr->sched_runtime < (1ULL << DL_SCALE)) + return false; + + /* + * Since we use the MSB for wrap-around and sign issues, make + * sure it's not set (mind that period can be equal to zero). + */ + if (attr->sched_deadline & (1ULL << 63) || + attr->sched_period & (1ULL << 63)) + return false; + + /* runtime <= deadline <= period (if period != 0) */ + if ((attr->sched_period != 0 && + attr->sched_period < attr->sched_deadline) || + attr->sched_deadline < attr->sched_runtime) + return false; + + return true; } + /* * check the target process has a UID that matches the current process's */ @@ -5583,7 +3521,10 @@ static int __sched_setscheduler(struct task_struct *p, const struct sched_attr *attr, bool user) { - int retval, oldprio, oldpolicy = -1, on_rq, running; + int retval, oldprio, oldpolicy = -1, queued, running; +#ifdef TJK_HMP + int new_effective_prio; +#endif int policy = attr->sched_policy; unsigned long flags; const struct sched_class *prev_class; @@ -5598,13 +3539,20 @@ static int __sched_setscheduler(struct task_struct *p, reset_on_fork = p->sched_reset_on_fork; policy = oldpolicy = p->policy; } else { - reset_on_fork = !!(policy & SCHED_RESET_ON_FORK); - policy &= ~SCHED_RESET_ON_FORK; + reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK); - if (policy != SCHED_FIFO && policy != SCHED_RR && + if (policy != SCHED_DEADLINE && + policy != SCHED_FIFO && policy != SCHED_RR && policy != SCHED_NORMAL && policy != SCHED_BATCH && - policy != SCHED_IDLE) + policy != SCHED_IDLE) { + pr_warn("%s %d:%s policy %d", __func__, p->pid, p->comm, policy); return -EINVAL; + } + } + + if (attr->sched_flags & ~(SCHED_FLAG_RESET_ON_FORK)) { + pr_warn("%s %d:%s sched_flags=%llu", __func__, p->pid, p->comm, attr->sched_flags); + return -EINVAL; } /* @@ -5612,20 +3560,29 @@ static int __sched_setscheduler(struct task_struct *p, * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL, * SCHED_BATCH and SCHED_IDLE is 0. */ - if (attr->sched_priority < 0 || - (p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) || - (!p->mm && attr->sched_priority > MAX_RT_PRIO-1)) + if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) || + (!p->mm && attr->sched_priority > MAX_RT_PRIO-1)) { + pr_warn("%s %d:%s sched_priority=%d", __func__, p->pid, p->comm, attr->sched_priority); return -EINVAL; - if (rt_policy(policy) != (attr->sched_priority != 0)) + } + if ((dl_policy(policy) && !__checkparam_dl(attr)) || + (rt_policy(policy) != (attr->sched_priority != 0))) { + pr_warn("%s %d:%s dl, rt sched_priority=%d", + __func__, p->pid, p->comm, attr->sched_priority); return -EINVAL; + } /* * Allow unprivileged RT tasks to decrease priority: */ if (user && !capable(CAP_SYS_NICE)) { if (fair_policy(policy)) { - if (!can_nice(p, attr->sched_nice)) + if (attr->sched_nice < task_nice(p) && + !can_nice(p, attr->sched_nice)) { + pr_warn("%s %d:%s sched_nice=%d", + __func__, p->pid, p->comm, attr->sched_nice); return -EPERM; + } } if (rt_policy(policy)) { @@ -5633,13 +3590,30 @@ static int __sched_setscheduler(struct task_struct *p, task_rlimit(p, RLIMIT_RTPRIO); /* can't set/change the rt policy */ - if (policy != p->policy && !rlim_rtprio) + if (policy != p->policy && !rlim_rtprio) { + pr_warn("%s %d:%s policy=%d %d %lu", + __func__, p->pid, p->comm, policy, p->policy, rlim_rtprio); return -EPERM; + } /* can't increase priority */ if (attr->sched_priority > p->rt_priority && - attr->sched_priority > rlim_rtprio) + attr->sched_priority > rlim_rtprio){ + pr_warn("%s %d:%s policy=%d %d %lu", __func__, p->pid, p->comm, + attr->sched_priority, p->rt_priority, rlim_rtprio); return -EPERM; + } + } + + /* + * Can't set/change SCHED_DEADLINE policy at all for now + * (safest behavior); in the future we would like to allow + * unprivileged DL tasks to increase their relative deadline + * or reduce their runtime (both ways reducing utilization) + */ + if (dl_policy(policy)) { + pr_warn("%s %d:%s dl policy=%d", __func__, p->pid, p->comm, policy); + return -EPERM; } /* @@ -5647,17 +3621,24 @@ static int __sched_setscheduler(struct task_struct *p, * SCHED_NORMAL if the RLIMIT_NICE would normally permit it. */ if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) { - if (!can_nice(p, TASK_NICE(p))) + if (!can_nice(p, task_nice(p))) { + pr_warn("%s %d:%s dl policy=%d", __func__, p->pid, p->comm, policy); return -EPERM; + } } /* can't change other user's priorities */ - if (!check_same_owner(p)) + if (!check_same_owner(p)) { + pr_warn("%s %d:%s check_same_owner", __func__, p->pid, p->comm); return -EPERM; + } /* Normal users shall not reset the sched_reset_on_fork flag */ - if (p->sched_reset_on_fork && !reset_on_fork) + if (p->sched_reset_on_fork && !reset_on_fork) { + pr_warn("%s %d:%s reset_on_fork=%d %d", + __func__, p->pid, p->comm, p->sched_reset_on_fork, reset_on_fork); return -EPERM; + } } if (user) { @@ -5684,21 +3665,25 @@ static int __sched_setscheduler(struct task_struct *p, } /* - * If not changing anything there's no need to proceed further: + * If not changing anything there's no need to proceed further, + * but store a possible modification of reset_on_fork. */ if (unlikely(policy == p->policy)) { - if (fair_policy(policy) && attr->sched_nice != TASK_NICE(p)) + if (fair_policy(policy) && attr->sched_nice != task_nice(p)) goto change; if (rt_policy(policy) && attr->sched_priority != p->rt_priority) goto change; + if (dl_policy(policy)) + goto change; + p->sched_reset_on_fork = reset_on_fork; task_rq_unlock(rq, p, &flags); return 0; } change: -#ifdef CONFIG_RT_GROUP_SCHED if (user) { +#ifdef CONFIG_RT_GROUP_SCHED /* * Do not allow realtime tasks into groups that have no runtime * assigned. @@ -5707,10 +3692,28 @@ static int __sched_setscheduler(struct task_struct *p, task_group(p)->rt_bandwidth.rt_runtime == 0 && !task_group_is_autogroup(task_group(p))) { task_rq_unlock(rq, p, &flags); + pr_warn("%s rt_runtime", __func__); return -EPERM; } - } #endif +#ifdef CONFIG_SMP + if (dl_bandwidth_enabled() && dl_policy(policy)) { + cpumask_t *span = rq->rd->span; + + /* + * Don't allow tasks with an affinity mask smaller than + * the entire root_domain to become SCHED_DEADLINE. We + * will also fail if there's no bandwidth available. + */ + if (!cpumask_subset(span, &p->cpus_allowed) || + rq->rd->dl_bw.bw == 0) { + task_rq_unlock(rq, p, &flags); + pr_warn("%s allowed", __func__); + return -EPERM; + } + } +#endif + } /* recheck policy now with rq lock held */ if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { @@ -5718,23 +3721,56 @@ static int __sched_setscheduler(struct task_struct *p, task_rq_unlock(rq, p, &flags); goto recheck; } - on_rq = p->on_rq; + + /* + * If setscheduling to SCHED_DEADLINE (or changing the parameters + * of a SCHED_DEADLINE task) we need to check if enough bandwidth + * is available. + */ + if ((dl_policy(policy) || dl_task(p)) && dl_overflow(p, policy, attr)) { + task_rq_unlock(rq, p, &flags); + pr_warn("%s deadline", __func__); + return -EBUSY; + } + + p->sched_reset_on_fork = reset_on_fork; + oldprio = p->prio; + + /* + * Take priority boosted tasks into account. If the new + * effective priority is unchanged, we just store the new + * normal parameters and do not touch the scheduler class and + * the runqueue. This will be done when the task deboost + * itself. + */ +#ifdef TJK_HMP + new_effective_prio = rt_mutex_get_effective_prio(p, newprio); + if (new_effective_prio == oldprio) { + __setscheduler_params(p, attr); + task_rq_unlock(rq, p, &flags); + return 0; + } +#endif + + queued = task_on_rq_queued(p); running = task_current(rq, p); - if (on_rq) + if (queued) dequeue_task(rq, p, 0); if (running) - p->sched_class->put_prev_task(rq, p); - - p->sched_reset_on_fork = reset_on_fork; + put_prev_task(rq, p); - oldprio = p->prio; prev_class = p->sched_class; - __setscheduler(rq, p, attr); + __setscheduler(rq, p, attr, true); if (running) p->sched_class->set_curr_task(rq); - if (on_rq) - enqueue_task(rq, p, 0); + if (queued) { + /* + * We enqueue to tail when the priority of a task is + * increased (user space view). + */ + enqueue_task(rq, p, oldprio <= p->prio ? ENQUEUE_HEAD : 0); + } check_class_changed(rq, p, prev_class, oldprio); task_rq_unlock(rq, p, &flags); @@ -5744,22 +3780,38 @@ static int __sched_setscheduler(struct task_struct *p, return 0; } +static int _sched_setscheduler(struct task_struct *p, int policy, + const struct sched_param *param, bool check) +{ + struct sched_attr attr = { + .sched_policy = policy, + .sched_priority = param->sched_priority, + .sched_nice = PRIO_TO_NICE(p->static_prio), + }; + + /* Fixup the legacy SCHED_RESET_ON_FORK hack. */ + if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) { + attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; + policy &= ~SCHED_RESET_ON_FORK; + attr.sched_policy = policy; + } + + return __sched_setscheduler(p, &attr, check); +} /** * sched_setscheduler - change the scheduling policy and/or RT priority of a thread. * @p: the task in question. * @policy: new policy. * @param: structure containing the new RT priority. * + * Return: 0 on success. An error code otherwise. + * * NOTE that the task may be already dead. */ int sched_setscheduler(struct task_struct *p, int policy, const struct sched_param *param) { - struct sched_attr attr = { - .sched_policy = policy, - .sched_priority = param->sched_priority - }; - return __sched_setscheduler(p, &attr, true); + return _sched_setscheduler(p, policy, param, true); } EXPORT_SYMBOL_GPL(sched_setscheduler); @@ -5779,15 +3831,13 @@ EXPORT_SYMBOL_GPL(sched_setattr); * current context has permission. For example, this is needed in * stop_machine(): we create temporary high priority worker threads, * but our caller might not have that capability. + * + * Return: 0 on success. An error code otherwise. */ int sched_setscheduler_nocheck(struct task_struct *p, int policy, const struct sched_param *param) { - struct sched_attr attr = { - .sched_policy = policy, - .sched_priority = param->sched_priority - }; - return __sched_setscheduler(p, &attr, false); + return _sched_setscheduler(p, policy, param, false); } static int @@ -5874,15 +3924,13 @@ static int sched_copy_attr(struct sched_attr __user *uattr, * XXX: do we want to be lenient like existing syscalls; or do we want * to be strict and return an error on out-of-bounds values? */ - attr->sched_nice = clamp(attr->sched_nice, -20, 19); + attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE); -out: - return ret; + return 0; err_size: put_user(sizeof(*attr), &uattr->size); - ret = -E2BIG; - goto out; + return -E2BIG; } /** @@ -5890,6 +3938,8 @@ static int sched_copy_attr(struct sched_attr __user *uattr, * @pid: the pid in question. * @policy: new policy. * @param: structure containing the new RT priority. + * + * Return: 0 on success. An error code otherwise. */ SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param) @@ -5905,16 +3955,19 @@ SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, * sys_sched_setparam - set/change the RT priority of a thread * @pid: the pid in question. * @param: structure containing the new RT priority. + * + * Return: 0 on success. An error code otherwise. */ SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param) { - return do_sched_setscheduler(pid, -1, param); + return do_sched_setscheduler(pid, SETPARAM_POLICY, param); } /** * sys_sched_setattr - same as above, but with extended sched_attr * @pid: the pid in question. - * @attr: structure containing the extended parameters. + * @uattr: structure containing the extended parameters. + * @flags: for future extension. */ SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, unsigned int, flags) @@ -5926,8 +3979,12 @@ SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, if (!uattr || pid < 0 || flags) return -EINVAL; - if (sched_copy_attr(uattr, &attr)) - return -EFAULT; + retval = sched_copy_attr(uattr, &attr); + if (retval) + return retval; + + if ((int)attr.sched_policy < 0) + return -EINVAL; rcu_read_lock(); retval = -ESRCH; @@ -5942,6 +3999,9 @@ SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, /** * sys_sched_getscheduler - get the policy (scheduling class) of a thread * @pid: the pid in question. + * + * Return: On success, the policy of the thread. Otherwise, a negative error + * code. */ SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) { @@ -5968,10 +4028,13 @@ SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) * sys_sched_getparam - get the RT priority of a thread * @pid: the pid in question. * @param: structure containing the RT priority. + * + * Return: On success, 0 and the RT priority is in @param. Otherwise, an error + * code. */ SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) { - struct sched_param lp; + struct sched_param lp = { .sched_priority = 0 }; struct task_struct *p; int retval; @@ -5988,7 +4051,8 @@ SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) if (retval) goto out_unlock; - lp.sched_priority = p->rt_priority; + if (task_has_rt_policy(p)) + lp.sched_priority = p->rt_priority; rcu_read_unlock(); /* @@ -6026,29 +4090,25 @@ static int sched_read_attr(struct sched_attr __user *uattr, for (; addr < end; addr++) { if (*addr) - goto err_size; + return -EFBIG; } attr->size = usize; } - ret = copy_to_user(uattr, attr, usize); + ret = copy_to_user(uattr, attr, attr->size); if (ret) return -EFAULT; -out: - return ret; - -err_size: - ret = -E2BIG; - goto out; + return 0; } /** - * sys_sched_getattr - same as above, but with extended "sched_param" + * sys_sched_getattr - similar to sched_getparam, but with sched_attr * @pid: the pid in question. - * @attr: structure containing the extended parameters. + * @uattr: structure containing the extended parameters. * @size: sizeof(attr) for fwd/bwd comp. + * @flags: for future extension. */ SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, unsigned int, size, unsigned int, flags) @@ -6074,10 +4134,14 @@ SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, goto out_unlock; attr.sched_policy = p->policy; - if (task_has_rt_policy(p)) + if (p->sched_reset_on_fork) + attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; + if (task_has_dl_policy(p)) + __getparam_dl(p, &attr); + else if (task_has_rt_policy(p)) attr.sched_priority = p->rt_priority; else - attr.sched_nice = TASK_NICE(p); + attr.sched_nice = task_nice(p); rcu_read_unlock(); @@ -6124,17 +4188,36 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) rcu_read_lock(); if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { rcu_read_unlock(); - goto out_unlock; + goto out_free_new_mask; } rcu_read_unlock(); } retval = security_task_setscheduler(p); if (retval) - goto out_unlock; + goto out_free_new_mask; + cpuset_cpus_allowed(p, cpus_allowed); cpumask_and(new_mask, in_mask, cpus_allowed); + + /* + * Since bandwidth control happens on root_domain basis, + * if admission test is enabled, we only admit -deadline + * tasks allowed to run on all the CPUs in the task's + * root_domain. + */ +#ifdef CONFIG_SMP + if (task_has_dl_policy(p) && dl_bandwidth_enabled()) { + rcu_read_lock(); + if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) { + retval = -EBUSY; + rcu_read_unlock(); + goto out_free_new_mask; + } + rcu_read_unlock(); + } +#endif again: retval = set_cpus_allowed_ptr(p, new_mask); @@ -6150,7 +4233,7 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) goto again; } } -out_unlock: +out_free_new_mask: free_cpumask_var(new_mask); out_free_cpus_allowed: free_cpumask_var(cpus_allowed); @@ -6175,6 +4258,8 @@ static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, * @pid: pid of the process * @len: length in bytes of the bitmask pointed to by user_mask_ptr * @user_mask_ptr: user-space pointer to the new cpu mask + * + * Return: 0 on success. An error code otherwise. */ SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len, unsigned long __user *, user_mask_ptr) @@ -6224,6 +4309,8 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask) * @pid: pid of the process * @len: length in bytes of the bitmask pointed to by user_mask_ptr * @user_mask_ptr: user-space pointer to hold the current cpu mask + * + * Return: 0 on success. An error code otherwise. */ SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, unsigned long __user *, user_mask_ptr) @@ -6258,6 +4345,8 @@ SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, * * This function yields the current CPU to other tasks. If there are no * other threads running on this CPU then this function will return. + * + * Return: 0. */ SYSCALL_DEFINE0(sched_yield) { @@ -6383,12 +4472,12 @@ EXPORT_SYMBOL(yield); * It's the caller's job to ensure that the target task struct * can't go away on us before we can do any checks. * - * Returns: + * Return: * true (>0) if we indeed boosted the target task. * false (0) if we failed to boost the target. * -ESRCH if there's no task to yield to. */ -bool __sched yield_to(struct task_struct *p, bool preempt) +int __sched yield_to(struct task_struct *p, bool preempt) { struct task_struct *curr = current; struct rq *rq, *p_rq; @@ -6410,7 +4499,7 @@ bool __sched yield_to(struct task_struct *p, bool preempt) } double_rq_lock(rq, p_rq); - while (task_rq(p) != p_rq) { + if (task_rq(p) != p_rq) { double_rq_unlock(rq, p_rq); goto again; } @@ -6432,7 +4521,7 @@ bool __sched yield_to(struct task_struct *p, bool preempt) * fairness. */ if (preempt && rq != p_rq) - resched_task(p_rq->curr); + resched_curr(p_rq); } out_unlock: @@ -6481,14 +4570,14 @@ long __sched io_schedule_timeout(long timeout) delayacct_blkio_end(); return ret; } -EXPORT_SYMBOL(io_schedule_timeout); /** * sys_sched_get_priority_max - return maximum RT priority. * @policy: scheduling class. * - * this syscall returns the maximum rt_priority that can be used - * by a given scheduling class. + * Return: On success, this syscall returns the maximum + * rt_priority that can be used by a given scheduling class. + * On failure, a negative error code is returned. */ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) { @@ -6499,6 +4588,7 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) case SCHED_RR: ret = MAX_USER_RT_PRIO-1; break; + case SCHED_DEADLINE: case SCHED_NORMAL: case SCHED_BATCH: case SCHED_IDLE: @@ -6512,8 +4602,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) * sys_sched_get_priority_min - return minimum RT priority. * @policy: scheduling class. * - * this syscall returns the minimum rt_priority that can be used - * by a given scheduling class. + * Return: On success, this syscall returns the minimum + * rt_priority that can be used by a given scheduling class. + * On failure, a negative error code is returned. */ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) { @@ -6524,6 +4615,7 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) case SCHED_RR: ret = 1; break; + case SCHED_DEADLINE: case SCHED_NORMAL: case SCHED_BATCH: case SCHED_IDLE: @@ -6539,6 +4631,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) * * this syscall writes the default timeslice value of a given process * into the user-space timespec buffer. A value of '0' means infinity. + * + * Return: On success, 0 and the timeslice is in @interval. Otherwise, + * an error code. */ SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, struct timespec __user *, interval) @@ -6564,7 +4659,9 @@ SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, goto out_unlock; rq = task_rq_lock(p, &flags); - time_slice = p->sched_class->get_rr_interval(rq, p); + time_slice = 0; + if (p->sched_class->get_rr_interval) + time_slice = p->sched_class->get_rr_interval(rq, p); task_rq_unlock(rq, p, &flags); rcu_read_unlock(); @@ -6625,7 +4722,7 @@ void show_state_filter(unsigned long state_filter) " task PC stack pid father\n"); #endif rcu_read_lock(); - do_each_thread(g, p) { + for_each_process_thread(g, p) { /* * reset the NMI-timeout, listing all files on a slow * console might take a lot of time: @@ -6633,11 +4730,11 @@ void show_state_filter(unsigned long state_filter) touch_nmi_watchdog(); if (!state_filter || (p->state & state_filter)) sched_show_task(p); - } while_each_thread(g, p); + } touch_all_softlockup_watchdogs(); -#ifdef CONFIG_SYSRQ_SCHED_DEBUG +#ifdef CONFIG_SCHED_DEBUG sysrq_sched_debug_show(); #endif rcu_read_unlock(); @@ -6648,7 +4745,7 @@ void show_state_filter(unsigned long state_filter) debug_show_all_locks(); } -void __cpuinit init_idle_bootup_task(struct task_struct *idle) +void init_idle_bootup_task(struct task_struct *idle) { idle->sched_class = &idle_sched_class; } @@ -6661,22 +4758,14 @@ void __cpuinit init_idle_bootup_task(struct task_struct *idle) * NOTE: this function does not set the idle thread's NEED_RESCHED * flag, to make booting more robust. */ -void __cpuinit init_idle(struct task_struct *idle, int cpu) +void init_idle(struct task_struct *idle, int cpu) { struct rq *rq = cpu_rq(cpu); unsigned long flags; - u64 mark_start; raw_spin_lock_irqsave(&rq->lock, flags); - mark_start = orig_mark_start(idle); - - __sched_fork(idle); - /* - * Restore idle thread's original mark_start as we rely on it being - * correct for maintaining per-cpu counters, curr/prev_runnable_sum. - */ - restore_orig_mark_start(idle, mark_start); + __sched_fork(0, idle); idle->state = TASK_RUNNING; idle->se.exec_start = sched_clock(); @@ -6696,6 +4785,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu) rcu_read_unlock(); rq->curr = rq->idle = idle; + idle->on_rq = TASK_ON_RQ_QUEUED; #if defined(CONFIG_SMP) idle->on_cpu = 1; #endif @@ -6716,6 +4806,33 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu) } #ifdef CONFIG_SMP +/* + * move_queued_task - move a queued task to new rq. + * + * Returns (locked) new rq. Old rq's lock is released. + */ +static struct rq *move_queued_task(struct task_struct *p, int new_cpu) +{ + struct rq *rq = task_rq(p); + + lockdep_assert_held(&rq->lock); + + dequeue_task(rq, p, 0); + p->on_rq = TASK_ON_RQ_MIGRATING; + set_task_cpu(p, new_cpu); + raw_spin_unlock(&rq->lock); + + rq = cpu_rq(new_cpu); + + raw_spin_lock(&rq->lock); + BUG_ON(task_cpu(p) != new_cpu); + p->on_rq = TASK_ON_RQ_QUEUED; + enqueue_task(rq, p, 0); + check_preempt_curr(rq, p, 0); + + return rq; +} + void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) { if (p->sched_class && p->sched_class->set_cpus_allowed) @@ -6772,14 +4889,15 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) goto out; dest_cpu = cpumask_any_and(cpu_active_mask, new_mask); - if (p->on_rq) { + if (task_running(rq, p) || p->state == TASK_WAKING) { struct migration_arg arg = { p, dest_cpu }; /* Need help from migration thread: drop lock and wait. */ task_rq_unlock(rq, p, &flags); stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); tlb_migrate_finish(p->mm); return 0; - } + } else if (task_on_rq_queued(p)) + rq = move_queued_task(p, dest_cpu); out: task_rq_unlock(rq, p, &flags); @@ -6800,21 +4918,20 @@ EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); */ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) { - struct rq *rq_dest, *rq_src; - bool moved = false; + struct rq *rq; int ret = 0; if (unlikely(!cpu_active(dest_cpu))) return ret; - rq_src = cpu_rq(src_cpu); - rq_dest = cpu_rq(dest_cpu); + rq = cpu_rq(src_cpu); raw_spin_lock(&p->pi_lock); - double_rq_lock(rq_src, rq_dest); + raw_spin_lock(&rq->lock); /* Already moved. */ if (task_cpu(p) != src_cpu) goto done; + /* Affinity changed (again). */ if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) goto fail; @@ -6823,34 +4940,64 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) * If we're not on a rq, the next wake-up will ensure we're * placed properly. */ - if (p->on_rq) { - dequeue_task(rq_src, p, 0); - set_task_cpu(p, dest_cpu); - enqueue_task(rq_dest, p, 0); - check_preempt_curr(rq_dest, p, 0); - moved = true; - } + if (task_on_rq_queued(p)) + rq = move_queued_task(p, dest_cpu); done: ret = 1; fail: - double_rq_unlock(rq_src, rq_dest); + raw_spin_unlock(&rq->lock); raw_spin_unlock(&p->pi_lock); - if (moved && !same_freq_domain(src_cpu, dest_cpu)) { - check_for_freq_change(rq_src); - check_for_freq_change(rq_dest); - } - if (moved && task_notify_on_migrate(p)) { - struct migration_notify_data mnd; - - mnd.src_cpu = src_cpu; - mnd.dest_cpu = dest_cpu; - mnd.load = pct_task_load(p); - atomic_notifier_call_chain(&migration_notifier_head, - 0, (void *)&mnd); - } return ret; } +#ifdef CONFIG_NUMA_BALANCING +/* Migrate current task p to target_cpu */ +int migrate_task_to(struct task_struct *p, int target_cpu) +{ + struct migration_arg arg = { p, target_cpu }; + int curr_cpu = task_cpu(p); + + if (curr_cpu == target_cpu) + return 0; + + if (!cpumask_test_cpu(target_cpu, tsk_cpus_allowed(p))) + return -EINVAL; + + /* TODO: This is not properly updating schedstats */ + + trace_sched_move_numa(p, curr_cpu, target_cpu); + return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg); +} + +/* + * Requeue a task on a given node and accurately track the number of NUMA + * tasks on the runqueues + */ +void sched_setnuma(struct task_struct *p, int nid) +{ + struct rq *rq; + unsigned long flags; + bool queued, running; + + rq = task_rq_lock(p, &flags); + queued = task_on_rq_queued(p); + running = task_current(rq, p); + + if (queued) + dequeue_task(rq, p, 0); + if (running) + put_prev_task(rq, p); + + p->numa_preferred_nid = nid; + + if (running) + p->sched_class->set_curr_task(rq); + if (queued) + enqueue_task(rq, p, 0); + task_rq_unlock(rq, p, &flags); +} +#endif + /* * migration_cpu_stop - this will be executed by a highprio stopper thread * and performs thread migration by bumping thread off CPU then @@ -6865,6 +5012,12 @@ static int migration_cpu_stop(void *data) * be on another cpu but it doesn't matter. */ local_irq_disable(); + /* + * We need to explicitly wake pending tasks before running + * __migrate_task() such that we will not miss enforcing cpus_allowed + * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. + */ + sched_ttwu_pending(); __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu); local_irq_enable(); return 0; @@ -6882,8 +5035,10 @@ void idle_task_exit(void) BUG_ON(cpu_online(smp_processor_id())); - if (mm != &init_mm) + if (mm != &init_mm) { switch_mm(mm, &init_mm, current); + finish_arch_post_lock_switch(); + } mmdrop(mm); } @@ -6901,6 +5056,22 @@ static void calc_load_migrate(struct rq *rq) atomic_long_add(delta, &calc_load_tasks); } +static void put_prev_task_fake(struct rq *rq, struct task_struct *prev) +{ +} + +static const struct sched_class fake_sched_class = { + .put_prev_task = put_prev_task_fake, +}; + +static struct task_struct fake_task = { + /* + * Avoid pull_{rt,dl}_task() + */ + .prio = MAX_PRIO + 1, + .sched_class = &fake_sched_class, +}; + /* * Migrate all tasks from the rq, sleeping tasks will be migrated by * try_to_wake_up()->select_task_rq(). @@ -6926,6 +5097,13 @@ static void migrate_tasks(unsigned int dead_cpu) */ rq->stop = NULL; + /* + * put_prev_task() and pick_next_task() sched + * class method both need to have an up-to-date + * value of rq->clock[_task] + */ + update_rq_clock(rq); + for ( ; ; ) { /* * There's this thread running, bail when that's the only @@ -6934,7 +5112,7 @@ static void migrate_tasks(unsigned int dead_cpu) if (rq->nr_running == 1) break; - next = pick_next_task(rq); + next = pick_next_task(rq, &fake_task); BUG_ON(!next); next->sched_class->put_prev_task(rq, next); @@ -7021,10 +5199,63 @@ set_table_entry(struct ctl_table *entry, } } +static struct ctl_table * +sd_alloc_ctl_energy_table(struct sched_group_energy *sge) +{ + struct ctl_table *table = sd_alloc_ctl_entry(6); + + if (table == NULL) + return NULL; + + set_table_entry(&table[0], "nr_idle_states", &sge->nr_idle_states, + sizeof(int), 0644, proc_dointvec_minmax, false); + set_table_entry(&table[1], "idle_states", &sge->idle_states[0].power, + sge->nr_idle_states*sizeof(struct idle_state), 0644, + proc_doulongvec_minmax, false); + set_table_entry(&table[2], "nr_idle_states_below", &sge->nr_idle_states_below, + sizeof(int), 0644, proc_dointvec_minmax, false); + set_table_entry(&table[3], "nr_cap_states", &sge->nr_cap_states, + sizeof(int), 0644, proc_dointvec_minmax, false); + set_table_entry(&table[4], "cap_states", &sge->cap_states[0].cap, + sge->nr_cap_states*sizeof(struct capacity_state), 0644, + proc_doulongvec_minmax, false); + + return table; +} + +static struct ctl_table * +sd_alloc_ctl_group_table(struct sched_group *sg) +{ + struct ctl_table *table = sd_alloc_ctl_entry(2); + + if (table == NULL) + return NULL; + + table->procname = kstrdup("energy", GFP_KERNEL); + table->mode = 0555; + table->child = sd_alloc_ctl_energy_table(sg->sge); + + return table; +} + static struct ctl_table * sd_alloc_ctl_domain_table(struct sched_domain *sd) { - struct ctl_table *table = sd_alloc_ctl_entry(13); + struct ctl_table *table; + unsigned int nr_entries = 14; + + int i = 0; + struct sched_group *sg = sd->groups; + + if (sg->sge) { + int nr_sgs = 0; + + do {} while (nr_sgs++, sg = sg->next, sg != sd->groups); + + nr_entries += nr_sgs; + } + + table = sd_alloc_ctl_entry(nr_entries); if (table == NULL) return NULL; @@ -7052,14 +5283,29 @@ sd_alloc_ctl_domain_table(struct sched_domain *sd) sizeof(int), 0644, proc_dointvec_minmax, false); set_table_entry(&table[10], "flags", &sd->flags, sizeof(int), 0644, proc_dointvec_minmax, false); - set_table_entry(&table[11], "name", sd->name, + set_table_entry(&table[11], "max_newidle_lb_cost", + &sd->max_newidle_lb_cost, + sizeof(long), 0644, proc_doulongvec_minmax, false); + set_table_entry(&table[12], "name", sd->name, CORENAME_MAX_SIZE, 0444, proc_dostring, false); - /* &table[12] is terminator */ + sg = sd->groups; + if (sg->sge) { + char buf[32]; + struct ctl_table *entry = &table[13]; + + do { + snprintf(buf, 32, "group%d", i); + entry->procname = kstrdup(buf, GFP_KERNEL); + entry->mode = 0555; + entry->child = sd_alloc_ctl_group_table(sg); + } while (entry++, i++, sg = sg->next, sg != sd->groups); + } + /* &table[nr_entries-1] is terminator */ return table; } -static ctl_table *sd_alloc_ctl_cpu_table(int cpu) +static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu) { struct ctl_table *entry, *table; struct sched_domain *sd; @@ -7161,7 +5407,7 @@ static void set_rq_offline(struct rq *rq) * migration_call - callback that gets triggered when a CPU is added. * Here we can start up the necessary migration thread for the new CPU. */ -static int __cpuinit +static int migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) { int cpu = (long)hcpu; @@ -7171,9 +5417,6 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) switch (action & ~CPU_TASKS_FROZEN) { case CPU_UP_PREPARE: - raw_spin_lock_irqsave(&rq->lock, flags); - set_window_start(rq); - raw_spin_unlock_irqrestore(&rq->lock, flags); rq->calc_load_update = calc_load_update; break; @@ -7193,8 +5436,6 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) sched_ttwu_pending(); /* Update our root-domain */ raw_spin_lock_irqsave(&rq->lock, flags); - migrate_sync_cpu(cpu); - if (rq->rd) { BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); set_rq_offline(rq); @@ -7205,7 +5446,6 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) break; case CPU_DEAD: - clear_hmp_request(cpu); calc_load_migrate(rq); break; #endif @@ -7221,15 +5461,25 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) * happens before everything else. This has to be lower priority than * the notifier in the perf_event subsystem, though. */ -static struct notifier_block __cpuinitdata migration_notifier = { +static struct notifier_block migration_notifier = { .notifier_call = migration_call, .priority = CPU_PRI_MIGRATION, }; -static int __cpuinit sched_cpu_active(struct notifier_block *nfb, +static void __cpuinit set_cpu_rq_start_time(void) +{ + int cpu = smp_processor_id(); + struct rq *rq = cpu_rq(cpu); + rq->age_stamp = sched_clock_cpu(cpu); +} + +static int sched_cpu_active(struct notifier_block *nfb, unsigned long action, void *hcpu) { switch (action & ~CPU_TASKS_FROZEN) { + case CPU_STARTING: + set_cpu_rq_start_time(); + return NOTIFY_OK; case CPU_DOWN_FAILED: set_cpu_active((long)hcpu, true); return NOTIFY_OK; @@ -7238,16 +5488,39 @@ static int __cpuinit sched_cpu_active(struct notifier_block *nfb, } } -static int __cpuinit sched_cpu_inactive(struct notifier_block *nfb, +static int sched_cpu_inactive(struct notifier_block *nfb, unsigned long action, void *hcpu) { + unsigned long flags; + long cpu = (long)hcpu; + struct dl_bw *dl_b; + switch (action & ~CPU_TASKS_FROZEN) { case CPU_DOWN_PREPARE: - set_cpu_active((long)hcpu, false); + set_cpu_active(cpu, false); + + /* explicitly allow suspend */ + if (!(action & CPU_TASKS_FROZEN)) { + bool overflow; + int cpus; + + rcu_read_lock_sched(); + dl_b = dl_bw_of(cpu); + + raw_spin_lock_irqsave(&dl_b->lock, flags); + cpus = dl_bw_cpus(cpu); + overflow = __dl_overflow(dl_b, cpus, 0, 0); + raw_spin_unlock_irqrestore(&dl_b->lock, flags); + + rcu_read_unlock_sched(); + + if (overflow) + return notifier_from_errno(-EBUSY); + } return NOTIFY_OK; - default: - return NOTIFY_DONE; } + + return NOTIFY_DONE; } static int __init migration_init(void) @@ -7329,18 +5602,6 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, break; } - /* - * Even though we initialize ->power to something semi-sane, - * we leave power_orig unset. This allows us to detect if - * domain iteration is still funny without causing /0 traps. - */ - if (!group->sgp->power_orig) { - printk(KERN_CONT "\n"); - printk(KERN_ERR "ERROR: domain->cpu_power not " - "set\n"); - break; - } - if (!cpumask_weight(sched_group_cpus(group))) { printk(KERN_CONT "\n"); printk(KERN_ERR "ERROR: empty group\n"); @@ -7359,9 +5620,9 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group)); printk(KERN_CONT " %s", str); - if (group->sgp->power != SCHED_CAPACITY_SCALE) { - printk(KERN_CONT " (cpu_power = %d)", - group->sgp->power); + if (group->sgc->capacity != SCHED_CAPACITY_SCALE) { + printk(KERN_CONT " (cpu_capacity = %ld)", + group->sgc->capacity); } group = group->next; @@ -7419,8 +5680,10 @@ static int sd_degenerate(struct sched_domain *sd) SD_BALANCE_NEWIDLE | SD_BALANCE_FORK | SD_BALANCE_EXEC | - SD_SHARE_CPUPOWER | - SD_SHARE_PKG_RESOURCES)) { + SD_SHARE_CPUCAPACITY | + SD_SHARE_PKG_RESOURCES | + SD_SHARE_POWERDOMAIN | + SD_SHARE_CAP_STATES)) { if (sd->groups != sd->groups->next) return 0; } @@ -7449,8 +5712,11 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) SD_BALANCE_NEWIDLE | SD_BALANCE_FORK | SD_BALANCE_EXEC | - SD_SHARE_CPUPOWER | - SD_SHARE_PKG_RESOURCES); + SD_SHARE_CPUCAPACITY | + SD_SHARE_PKG_RESOURCES | + SD_PREFER_SIBLING | + SD_SHARE_POWERDOMAIN | + SD_SHARE_CAP_STATES); if (nr_node_ids == 1) pflags &= ~SD_SERIALIZE; } @@ -7465,6 +5731,8 @@ static void free_rootdomain(struct rcu_head *rcu) struct root_domain *rd = container_of(rcu, struct root_domain, rcu); cpupri_cleanup(&rd->cpupri); + cpudl_cleanup(&rd->cpudl); + free_cpumask_var(rd->dlo_mask); free_cpumask_var(rd->rto_mask); free_cpumask_var(rd->online); free_cpumask_var(rd->span); @@ -7487,7 +5755,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd) cpumask_clear_cpu(rq->cpu, old_rd->span); /* - * If we dont want to free the old_rt yet then + * If we dont want to free the old_rd yet then * set old_rd to NULL to skip the freeing later * in this function: */ @@ -7516,8 +5784,14 @@ static int init_rootdomain(struct root_domain *rd) goto out; if (!alloc_cpumask_var(&rd->online, GFP_KERNEL)) goto free_span; - if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL)) + if (!alloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL)) goto free_online; + if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL)) + goto free_dlo_mask; + + init_dl_bw(&rd->dl_bw); + if (cpudl_init(&rd->cpudl) != 0) + goto free_dlo_mask; if (cpupri_init(&rd->cpupri) != 0) goto free_rto_mask; @@ -7525,6 +5799,8 @@ static int init_rootdomain(struct root_domain *rd) free_rto_mask: free_cpumask_var(rd->rto_mask); +free_dlo_mask: + free_cpumask_var(rd->dlo_mask); free_online: free_cpumask_var(rd->online); free_span: @@ -7562,7 +5838,7 @@ static struct root_domain *alloc_rootdomain(void) return rd; } -static void free_sched_groups(struct sched_group *sg, int free_sgp) +static void free_sched_groups(struct sched_group *sg, int free_sgc) { struct sched_group *tmp, *first; @@ -7573,8 +5849,11 @@ static void free_sched_groups(struct sched_group *sg, int free_sgp) do { tmp = sg->next; - if (free_sgp && atomic_dec_and_test(&sg->sgp->ref)) - kfree(sg->sgp); + if (free_sgc && atomic_dec_and_test(&sg->sgc->ref)) + kfree(sg->sgc); + + if (free_sgc && atomic_dec_and_test(&sg->sge->ref)) + kfree(sg->sge); kfree(sg); sg = tmp; @@ -7592,7 +5871,8 @@ static void free_sched_domain(struct rcu_head *rcu) if (sd->flags & SD_OVERLAP) { free_sched_groups(sd->groups, 1); } else if (atomic_dec_and_test(&sd->groups->ref)) { - kfree(sd->groups->sgp); + kfree(sd->groups->sgc); + kfree(sd->groups->sge); kfree(sd->groups); } kfree(sd); @@ -7619,19 +5899,45 @@ static void destroy_sched_domains(struct sched_domain *sd, int cpu) * two cpus are in the same cache domain, see cpus_share_cache(). */ DEFINE_PER_CPU(struct sched_domain *, sd_llc); +DEFINE_PER_CPU(int, sd_llc_size); DEFINE_PER_CPU(int, sd_llc_id); +DEFINE_PER_CPU(struct sched_domain *, sd_numa); +DEFINE_PER_CPU(struct sched_domain *, sd_busy); +DEFINE_PER_CPU(struct sched_domain *, sd_asym); +DEFINE_PER_CPU(struct sched_domain *, sd_ea); static void update_top_cache_domain(int cpu) { struct sched_domain *sd; + struct sched_domain *busy_sd = NULL, *ea_sd = NULL; int id = cpu; + int size = 1; sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES); - if (sd) + if (sd) { id = cpumask_first(sched_domain_span(sd)); + size = cpumask_weight(sched_domain_span(sd)); + busy_sd = sd->parent; /* sd_busy */ + } + rcu_assign_pointer(per_cpu(sd_busy, cpu), busy_sd); rcu_assign_pointer(per_cpu(sd_llc, cpu), sd); + per_cpu(sd_llc_size, cpu) = size; per_cpu(sd_llc_id, cpu) = id; + + sd = lowest_flag_domain(cpu, SD_NUMA); + rcu_assign_pointer(per_cpu(sd_numa, cpu), sd); + + sd = highest_flag_domain(cpu, SD_ASYM_PACKING); + rcu_assign_pointer(per_cpu(sd_asym, cpu), sd); + + for_each_domain(cpu, sd) { + if (sd->groups->sge) + ea_sd = sd; + else + break; + } + rcu_assign_pointer(per_cpu(sd_ea, cpu), ea_sd); } /* @@ -7643,7 +5949,6 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) { struct rq *rq = cpu_rq(cpu); struct sched_domain *tmp; - unsigned long next_balance = rq->next_balance; /* Remove the sched domains which do not contribute to scheduling. */ for (tmp = sd; tmp; ) { @@ -7655,6 +5960,13 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) tmp->parent = parent->parent; if (parent->parent) parent->parent->child = tmp; + /* + * Transfer SD_PREFER_SIBLING down in case of a + * degenerate parent; the spans match for this + * so the property transfers. + */ + if (parent->flags & SD_PREFER_SIBLING) + tmp->flags |= SD_PREFER_SIBLING; destroy_sched_domain(parent, cpu); } else tmp = tmp->parent; @@ -7668,17 +5980,6 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) sd->child = NULL; } - for (tmp = sd; tmp; ) { - unsigned long interval; - - interval = msecs_to_jiffies(tmp->balance_interval); - if (time_after(next_balance, tmp->last_balance + interval)) - next_balance = tmp->last_balance + interval; - - tmp = tmp->parent; - } - rq->next_balance = next_balance; - sched_domain_debug(sd, cpu); rq_attach_root(rq, rd); @@ -7702,17 +6003,6 @@ static int __init isolated_cpu_setup(char *str) __setup("isolcpus=", isolated_cpu_setup); -static const struct cpumask *cpu_cpu_mask(int cpu) -{ - return cpumask_of_node(cpu_to_node(cpu)); -} - -struct sd_data { - struct sched_domain **__percpu sd; - struct sched_group **__percpu sg; - struct sched_group_power **__percpu sgp; -}; - struct s_data { struct sched_domain ** __percpu sd; struct root_domain *rd; @@ -7725,21 +6015,6 @@ enum s_alloc { sa_none, }; -struct sched_domain_topology_level; - -typedef struct sched_domain *(*sched_domain_init_f)(struct sched_domain_topology_level *tl, int cpu); -typedef const struct cpumask *(*sched_domain_mask_f)(int cpu); - -#define SDTL_OVERLAP 0x01 - -struct sched_domain_topology_level { - sched_domain_init_f init; - sched_domain_mask_f mask; - int flags; - int numa_level; - struct sd_data data; -}; - /* * Build an iteration mask that can exclude certain CPUs from the upwards * domain traversal. @@ -7785,7 +6060,7 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu) const struct cpumask *span = sched_domain_span(sd); struct cpumask *covered = sched_domains_tmpmask; struct sd_data *sdd = sd->private; - struct sched_domain *child; + struct sched_domain *sibling; int i; cpumask_clear(covered); @@ -7796,10 +6071,10 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu) if (cpumask_test_cpu(i, covered)) continue; - child = *per_cpu_ptr(sdd->sd, i); + sibling = *per_cpu_ptr(sdd->sd, i); /* See the comment near build_group_mask(). */ - if (!cpumask_test_cpu(i, sched_domain_span(child))) + if (!cpumask_test_cpu(i, sched_domain_span(sibling))) continue; sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(), @@ -7809,24 +6084,26 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu) goto fail; sg_span = sched_group_cpus(sg); - if (child->child) { - child = child->child; - cpumask_copy(sg_span, sched_domain_span(child)); - } else + if (sibling->child) + cpumask_copy(sg_span, sched_domain_span(sibling->child)); + else cpumask_set_cpu(i, sg_span); cpumask_or(covered, covered, sg_span); - sg->sgp = *per_cpu_ptr(sdd->sgp, i); - if (atomic_inc_return(&sg->sgp->ref) == 1) + sg->sgc = *per_cpu_ptr(sdd->sgc, i); + if (atomic_inc_return(&sg->sgc->ref) == 1) build_group_mask(sd, sg); /* - * Initialize sgp->power such that even if we mess up the + * Initialize sgc->capacity such that even if we mess up the * domains and no possible iteration will get us here, we won't * die on a /0 trap. */ - sg->sgp->power = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span); + sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span); + sg->sgc->max_capacity = SCHED_CAPACITY_SCALE; + + sg->sge = *per_cpu_ptr(sdd->sge, i); /* * Make sure the first group of this domain contains the @@ -7864,8 +6141,9 @@ static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg) if (sg) { *sg = *per_cpu_ptr(sdd->sg, cpu); - (*sg)->sgp = *per_cpu_ptr(sdd->sgp, cpu); - atomic_set(&(*sg)->sgp->ref, 1); /* for claim_allocations */ + (*sg)->sgc = *per_cpu_ptr(sdd->sgc, cpu); + atomic_set(&(*sg)->sgc->ref, 1); /* for claim_allocations */ + (*sg)->sge = *per_cpu_ptr(sdd->sge, cpu); } return cpu; @@ -7874,7 +6152,7 @@ static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg) /* * build_sched_groups will build a circular linked list of the groups * covered by the given span, and will set each group's ->cpumask correctly, - * and ->cpu_power to 0. + * and ->cpu_capacity to 0. * * Assumes the sched_domain tree is fully constructed */ @@ -7890,7 +6168,7 @@ build_sched_groups(struct sched_domain *sd, int cpu) get_group(cpu, sdd, &sd->groups); atomic_inc(&sd->groups->ref); - if (cpu != cpumask_first(sched_domain_span(sd))) + if (cpu != cpumask_first(span)) return 0; lockdep_assert_held(&sched_domains_mutex); @@ -7900,14 +6178,12 @@ build_sched_groups(struct sched_domain *sd, int cpu) for_each_cpu(i, span) { struct sched_group *sg; - int group = get_group(i, sdd, &sg); - int j; + int group, j; if (cpumask_test_cpu(i, covered)) continue; - cpumask_clear(sched_group_cpus(sg)); - sg->sgp->power = 0; + group = get_group(i, sdd, &sg); cpumask_setall(sched_group_mask(sg)); for_each_cpu(j, span) { @@ -7930,20 +6206,20 @@ build_sched_groups(struct sched_domain *sd, int cpu) } /* - * Initialize sched groups cpu_power. + * Initialize sched groups cpu_capacity. * - * cpu_power indicates the capacity of sched group, which is used while + * cpu_capacity indicates the capacity of sched group, which is used while * distributing the load between different sched groups in a sched domain. - * Typically cpu_power for all the groups in a sched domain will be same unless - * there are asymmetries in the topology. If there are asymmetries, group - * having more cpu_power will pickup more load compared to the group having - * less cpu_power. + * Typically cpu_capacity for all the groups in a sched domain will be same + * unless there are asymmetries in the topology. If there are asymmetries, + * group having more cpu_capacity will pickup more load compared to the + * group having less cpu_capacity. */ -static void init_sched_groups_power(int cpu, struct sched_domain *sd) +static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) { struct sched_group *sg = sd->groups; - WARN_ON(!sd || !sg); + WARN_ON(!sg); do { sg->group_weight = cpumask_weight(sched_group_cpus(sg)); @@ -7953,47 +6229,72 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd) if (cpu != group_balance_cpu(sg)) return; - update_group_power(sd, cpu); - atomic_set(&sg->sgp->nr_busy_cpus, sg->group_weight); + update_group_capacity(sd, cpu); + atomic_set(&sg->sgc->nr_busy_cpus, sg->group_weight); } -int __weak arch_sd_sibling_asym_packing(void) +static void init_sched_energy(int cpu, struct sched_domain *sd, + struct sched_domain_topology_level *tl) { - return 0*SD_ASYM_PACKING; -} - -/* - * Initializers for schedule domains - * Non-inlined to reduce accumulated stack pressure in build_sched_domains() - */ + struct sched_group *sg = sd->groups; + struct sched_group_energy *sge = sg->sge; + sched_domain_energy_f fn = tl->energy; + struct cpumask *mask = sched_group_cpus(sg); + int nr_idle_states_below = 0; + if (fn && sd->child && !sd->child->groups->sge) { + pr_err("BUG: EAS setup broken for CPU%d\n", cpu); #ifdef CONFIG_SCHED_DEBUG -# define SD_INIT_NAME(sd, type) sd->name = #type -#else -# define SD_INIT_NAME(sd, type) do { } while (0) + pr_err(" energy data on %s but not on %s domain\n", + sd->name, sd->child->name); #endif + return; + } + + if (cpu != group_balance_cpu(sg)) + return; -#define SD_INIT_FUNC(type) \ -static noinline struct sched_domain * \ -sd_init_##type(struct sched_domain_topology_level *tl, int cpu) \ -{ \ - struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu); \ - *sd = SD_##type##_INIT; \ - SD_INIT_NAME(sd, type); \ - sd->private = &tl->data; \ - return sd; \ + if (!fn || !fn(cpu)) { + sg->sge = NULL; + return; + } + + atomic_set(&sg->sge->ref, 1); /* for claim_allocations */ + + if (cpumask_weight(mask) > 1) + check_sched_energy_data(cpu, fn, mask); + + /* Figure out the number of true cpuidle states below current group */ + sd = sd->child; + for_each_lower_domain(sd) { + nr_idle_states_below += sd->groups->sge->nr_idle_states; + + /* Disregard non-cpuidle 'active' idle states */ + if (sd->child) + nr_idle_states_below--; + } + + sge->nr_idle_states = fn(cpu)->nr_idle_states; + sge->nr_idle_states_below = nr_idle_states_below; + sge->nr_cap_states = fn(cpu)->nr_cap_states; + sge->idle_states = (struct idle_state *) + ((void *)&sge->cap_states + + sizeof(sge->cap_states)); + sge->cap_states = (struct capacity_state *) + ((void *)&sge->cap_states + + sizeof(sge->cap_states) + + sge->nr_idle_states * + sizeof(struct idle_state)); + memcpy(sge->idle_states, fn(cpu)->idle_states, + sge->nr_idle_states*sizeof(struct idle_state)); + memcpy(sge->cap_states, fn(cpu)->cap_states, + sge->nr_cap_states*sizeof(struct capacity_state)); } -SD_INIT_FUNC(CPU) -#ifdef CONFIG_SCHED_SMT - SD_INIT_FUNC(SIBLING) -#endif -#ifdef CONFIG_SCHED_MC - SD_INIT_FUNC(MC) -#endif -#ifdef CONFIG_SCHED_BOOK - SD_INIT_FUNC(BOOK) -#endif +/* + * Initializers for schedule domains + * Non-inlined to reduce accumulated stack pressure in build_sched_domains() + */ static int default_relax_domain_level = -1; int sched_domain_level_max; @@ -8078,97 +6379,165 @@ static void claim_allocations(int cpu, struct sched_domain *sd) if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref)) *per_cpu_ptr(sdd->sg, cpu) = NULL; - if (atomic_read(&(*per_cpu_ptr(sdd->sgp, cpu))->ref)) - *per_cpu_ptr(sdd->sgp, cpu) = NULL; -} + if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref)) + *per_cpu_ptr(sdd->sgc, cpu) = NULL; -#ifdef CONFIG_SCHED_SMT -static const struct cpumask *cpu_smt_mask(int cpu) -{ - return topology_thread_cpumask(cpu); + if (atomic_read(&(*per_cpu_ptr(sdd->sge, cpu))->ref)) + *per_cpu_ptr(sdd->sge, cpu) = NULL; } -#endif - -/* - * Topology list, bottom-up. - */ -static struct sched_domain_topology_level default_topology[] = { -#ifdef CONFIG_SCHED_SMT - { sd_init_SIBLING, cpu_smt_mask, }, -#endif -#ifdef CONFIG_SCHED_MC - { sd_init_MC, cpu_coregroup_mask, }, -#endif -#ifdef CONFIG_SCHED_BOOK - { sd_init_BOOK, cpu_book_mask, }, -#endif - { sd_init_CPU, cpu_cpu_mask, }, - { NULL, }, -}; - -static struct sched_domain_topology_level *sched_domain_topology = default_topology; #ifdef CONFIG_NUMA - static int sched_domains_numa_levels; static int *sched_domains_numa_distance; static struct cpumask ***sched_domains_numa_masks; static int sched_domains_curr_level; +#endif -static inline int sd_local_flags(int level) -{ - if (sched_domains_numa_distance[level] > RECLAIM_DISTANCE) - return 0; - - return SD_BALANCE_EXEC | SD_BALANCE_FORK | SD_WAKE_AFFINE; -} +/* + * SD_flags allowed in topology descriptions. + * + * SD_SHARE_CPUCAPACITY - describes SMT topologies + * SD_SHARE_PKG_RESOURCES - describes shared caches + * SD_NUMA - describes NUMA topologies + * SD_SHARE_POWERDOMAIN - describes shared power domain + * SD_SHARE_CAP_STATES - describes shared capacity states + * + * Odd one out: + * SD_ASYM_PACKING - describes SMT quirks + */ +#define TOPOLOGY_SD_FLAGS \ + (SD_SHARE_CPUCAPACITY | \ + SD_SHARE_PKG_RESOURCES | \ + SD_NUMA | \ + SD_ASYM_PACKING | \ + SD_SHARE_POWERDOMAIN | \ + SD_SHARE_CAP_STATES) static struct sched_domain * -sd_numa_init(struct sched_domain_topology_level *tl, int cpu) +sd_init(struct sched_domain_topology_level *tl, int cpu) { struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu); - int level = tl->numa_level; - int sd_weight = cpumask_weight( - sched_domains_numa_masks[level][cpu_to_node(cpu)]); + int sd_weight, sd_flags = 0; + +#ifdef CONFIG_NUMA + /* + * Ugly hack to pass state to sd_numa_mask()... + */ + sched_domains_curr_level = tl->numa_level; +#endif + + sd_weight = cpumask_weight(tl->mask(cpu)); + + if (tl->sd_flags) + sd_flags = (*tl->sd_flags)(); + if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS, + "wrong sd_flags in topology description\n")) + sd_flags &= ~TOPOLOGY_SD_FLAGS; *sd = (struct sched_domain){ .min_interval = sd_weight, .max_interval = 2*sd_weight, .busy_factor = 32, .imbalance_pct = 125, - .cache_nice_tries = 2, - .busy_idx = 3, - .idle_idx = 2, + + .cache_nice_tries = 0, + .busy_idx = 0, + .idle_idx = 0, .newidle_idx = 0, .wake_idx = 0, .forkexec_idx = 0, .flags = 1*SD_LOAD_BALANCE | 1*SD_BALANCE_NEWIDLE - | 0*SD_BALANCE_EXEC - | 0*SD_BALANCE_FORK - | 0*SD_BALANCE_WAKE - | 0*SD_WAKE_AFFINE - | 0*SD_SHARE_CPUPOWER + | 1*SD_BALANCE_EXEC + | 1*SD_BALANCE_FORK + | 1*SD_BALANCE_WAKE + | 1*SD_WAKE_AFFINE + | 0*SD_SHARE_CPUCAPACITY | 0*SD_SHARE_PKG_RESOURCES - | 1*SD_SERIALIZE + | 0*SD_SERIALIZE | 0*SD_PREFER_SIBLING - | sd_local_flags(level) + | 0*SD_NUMA + | sd_flags , + .last_balance = jiffies, .balance_interval = sd_weight, + .smt_gain = 0, + .max_newidle_lb_cost = 0, + .next_decay_max_lb_cost = jiffies, +#ifdef CONFIG_SCHED_DEBUG + .name = tl->name, +#endif }; - SD_INIT_NAME(sd, NUMA); - sd->private = &tl->data; /* - * Ugly hack to pass state to sd_numa_mask()... + * Convert topological properties into behaviour. */ - sched_domains_curr_level = tl->numa_level; + + if (sd->flags & SD_SHARE_CPUCAPACITY) { + sd->imbalance_pct = 110; + sd->smt_gain = 1178; /* ~15% */ + + } else if (sd->flags & SD_SHARE_PKG_RESOURCES) { + sd->imbalance_pct = 117; + sd->cache_nice_tries = 1; + sd->busy_idx = 2; + +#ifdef CONFIG_NUMA + } else if (sd->flags & SD_NUMA) { + sd->cache_nice_tries = 2; + sd->busy_idx = 3; + sd->idle_idx = 2; + + sd->flags |= SD_SERIALIZE; + if (sched_domains_numa_distance[tl->numa_level] > RECLAIM_DISTANCE) { + sd->flags &= ~(SD_BALANCE_EXEC | + SD_BALANCE_FORK | + SD_WAKE_AFFINE); + } + +#endif + } else { + sd->flags |= SD_PREFER_SIBLING; + sd->cache_nice_tries = 1; + sd->busy_idx = 2; + sd->idle_idx = 1; + } + + sd->private = &tl->data; return sd; } +/* + * Topology list, bottom-up. + */ +static struct sched_domain_topology_level default_topology[] = { +#ifdef CONFIG_SCHED_SMT + { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) }, +#endif +#ifdef CONFIG_SCHED_MC + { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) }, +#endif +#ifdef TJK_HMP + { sd_init_CPU, cpu_cpu_mask, }, +#endif + { NULL, }, +}; + +struct sched_domain_topology_level *sched_domain_topology = default_topology; + +#define for_each_sd_topology(tl) \ + for (tl = sched_domain_topology; tl->mask; tl++) + +void set_sched_topology(struct sched_domain_topology_level *tl) +{ + sched_domain_topology = tl; +} + +#ifdef CONFIG_NUMA + static const struct cpumask *sd_numa_mask(int cpu) { return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)]; @@ -8263,6 +6632,10 @@ static void sched_init_numa(void) if (!sched_debug()) break; } + + if (!level) + return; + /* * 'level' contains the number of unique distances, excluding the * identity distance node_distance(i,i). @@ -8312,7 +6685,10 @@ static void sched_init_numa(void) } } - tl = kzalloc((ARRAY_SIZE(default_topology) + level) * + /* Compute default topology size */ + for (i = 0; sched_domain_topology[i].mask; i++); + + tl = kzalloc((i + level + 1) * sizeof(struct sched_domain_topology_level), GFP_KERNEL); if (!tl) return; @@ -8320,18 +6696,19 @@ static void sched_init_numa(void) /* * Copy the default topology bits.. */ - for (i = 0; default_topology[i].init; i++) - tl[i] = default_topology[i]; + for (i = 0; sched_domain_topology[i].mask; i++) + tl[i] = sched_domain_topology[i]; /* * .. and append 'j' levels of NUMA goodness. */ for (j = 0; j < level; i++, j++) { tl[i] = (struct sched_domain_topology_level){ - .init = sd_numa_init, .mask = sd_numa_mask, + .sd_flags = cpu_numa_flags, .flags = SDTL_OVERLAP, .numa_level = j, + SD_INIT_NAME(NUMA) }; } @@ -8405,7 +6782,7 @@ static int __sdt_alloc(const struct cpumask *cpu_map) struct sched_domain_topology_level *tl; int j; - for (tl = sched_domain_topology; tl->init; tl++) { + for_each_sd_topology(tl) { struct sd_data *sdd = &tl->data; sdd->sd = alloc_percpu(struct sched_domain *); @@ -8416,14 +6793,28 @@ static int __sdt_alloc(const struct cpumask *cpu_map) if (!sdd->sg) return -ENOMEM; - sdd->sgp = alloc_percpu(struct sched_group_power *); - if (!sdd->sgp) + sdd->sgc = alloc_percpu(struct sched_group_capacity *); + if (!sdd->sgc) + return -ENOMEM; + + sdd->sge = alloc_percpu(struct sched_group_energy *); + if (!sdd->sge) return -ENOMEM; for_each_cpu(j, cpu_map) { struct sched_domain *sd; struct sched_group *sg; - struct sched_group_power *sgp; + struct sched_group_capacity *sgc; + struct sched_group_energy *sge; + sched_domain_energy_f fn = tl->energy; + unsigned int nr_idle_states = 0; + unsigned int nr_cap_states = 0; + + if (fn && fn(j)) { + nr_idle_states = fn(j)->nr_idle_states; + nr_cap_states = fn(j)->nr_cap_states; + BUG_ON(!nr_idle_states || !nr_cap_states); + } sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(), GFP_KERNEL, cpu_to_node(j)); @@ -8441,12 +6832,22 @@ static int __sdt_alloc(const struct cpumask *cpu_map) *per_cpu_ptr(sdd->sg, j) = sg; - sgp = kzalloc_node(sizeof(struct sched_group_power) + cpumask_size(), + sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(), GFP_KERNEL, cpu_to_node(j)); - if (!sgp) + if (!sgc) + return -ENOMEM; + + *per_cpu_ptr(sdd->sgc, j) = sgc; + + sge = kzalloc_node(sizeof(struct sched_group_energy) + + nr_idle_states*sizeof(struct idle_state) + + nr_cap_states*sizeof(struct capacity_state), + GFP_KERNEL, cpu_to_node(j)); + + if (!sge) return -ENOMEM; - *per_cpu_ptr(sdd->sgp, j) = sgp; + *per_cpu_ptr(sdd->sge, j) = sge; } } @@ -8458,7 +6859,7 @@ static void __sdt_free(const struct cpumask *cpu_map) struct sched_domain_topology_level *tl; int j; - for (tl = sched_domain_topology; tl->init; tl++) { + for_each_sd_topology(tl) { struct sd_data *sdd = &tl->data; for_each_cpu(j, cpu_map) { @@ -8473,24 +6874,27 @@ static void __sdt_free(const struct cpumask *cpu_map) if (sdd->sg) kfree(*per_cpu_ptr(sdd->sg, j)); - if (sdd->sgp) - kfree(*per_cpu_ptr(sdd->sgp, j)); + if (sdd->sgc) + kfree(*per_cpu_ptr(sdd->sgc, j)); + if (sdd->sge) + kfree(*per_cpu_ptr(sdd->sge, j)); } free_percpu(sdd->sd); sdd->sd = NULL; free_percpu(sdd->sg); sdd->sg = NULL; - free_percpu(sdd->sgp); - sdd->sgp = NULL; + free_percpu(sdd->sgc); + sdd->sgc = NULL; + free_percpu(sdd->sge); + sdd->sge = NULL; } } struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl, - struct s_data *d, const struct cpumask *cpu_map, - struct sched_domain_attr *attr, struct sched_domain *child, - int cpu) + const struct cpumask *cpu_map, struct sched_domain_attr *attr, + struct sched_domain *child, int cpu) { - struct sched_domain *sd = tl->init(tl, cpu); + struct sched_domain *sd = sd_init(tl, cpu); if (!sd) return child; @@ -8499,8 +6903,22 @@ struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl, sd->level = child->level + 1; sched_domain_level_max = max(sched_domain_level_max, sd->level); child->parent = sd; + sd->child = child; + + if (!cpumask_subset(sched_domain_span(child), + sched_domain_span(sd))) { + pr_err("BUG: arch topology borken\n"); +#ifdef CONFIG_SCHED_DEBUG + pr_err(" the %s domain not a subset of the %s domain\n", + child->name, sd->name); +#endif + /* Fixup, ensure @sd has at least @child cpus. */ + cpumask_or(sched_domain_span(sd), + sched_domain_span(sd), + sched_domain_span(child)); + } + } - sd->child = child; set_domain_attribute(sd, attr); return sd; @@ -8513,9 +6931,10 @@ struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl, static int build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr) { - enum s_alloc alloc_state = sa_none; + enum s_alloc alloc_state; struct sched_domain *sd; struct s_data d; + struct rq *rq; int i, ret = -ENOMEM; alloc_state = __visit_domain_allocation_hell(&d, cpu_map); @@ -8527,18 +6946,15 @@ static int build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_topology_level *tl; sd = NULL; - for (tl = sched_domain_topology; tl->init; tl++) { - sd = build_sched_domain(tl, &d, cpu_map, attr, sd, i); + for_each_sd_topology(tl) { + sd = build_sched_domain(tl, cpu_map, attr, sd, i); + if (tl == sched_domain_topology) + *per_cpu_ptr(d.sd, i) = sd; if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP)) sd->flags |= SD_OVERLAP; if (cpumask_equal(cpu_map, sched_domain_span(sd))) break; } - - while (sd->child) - sd = sd->child; - - *per_cpu_ptr(d.sd, i) = sd; } /* Build the groups for the domains */ @@ -8555,25 +6971,35 @@ static int build_sched_domains(const struct cpumask *cpu_map, } } - /* Calculate CPU power for physical packages and nodes */ + /* Calculate CPU capacity for physical packages and nodes */ for (i = nr_cpumask_bits-1; i >= 0; i--) { + struct sched_domain_topology_level *tl = sched_domain_topology; + if (!cpumask_test_cpu(i, cpu_map)) continue; - for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { + for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent, tl++) { + init_sched_energy(i, sd, tl); claim_allocations(i, sd); - init_sched_groups_power(i, sd); + init_sched_groups_capacity(i, sd); } } /* Attach the domains */ rcu_read_lock(); for_each_cpu(i, cpu_map) { + rq = cpu_rq(i); sd = *per_cpu_ptr(d.sd, i); cpu_attach_domain(sd, d.rd, i); + + if (rq->cpu_capacity_orig > rq->rd->max_cpu_capacity) + rq->rd->max_cpu_capacity = rq->cpu_capacity_orig; } rcu_read_unlock(); + rq = cpu_rq(cpumask_first(cpu_map)); + pr_info("Max cpu capacity: %lu\n", rq->rd->max_cpu_capacity); + ret = 0; error: __free_domain_allocs(&d, alloc_state, cpu_map); @@ -8597,7 +7023,7 @@ static cpumask_var_t fallback_doms; * cpu core maps. It is supposed to return 1 if the topology changed * or 0 if it stayed the same. */ -int __attribute__((weak)) arch_update_cpu_topology(void) +int __weak arch_update_cpu_topology(void) { return 0; } @@ -8733,8 +7159,9 @@ void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], ; } + n = ndoms_cur; if (doms_new == NULL) { - ndoms_cur = 0; + n = 0; doms_new = &fallback_doms; cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map); WARN_ON_ONCE(dattr_new); @@ -8742,7 +7169,7 @@ void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], /* Build new domains */ for (i = 0; i < ndoms_new; i++) { - for (j = 0; j < ndoms_cur && !new_topology; j++) { + for (j = 0; j < n && !new_topology; j++) { if (cpumask_equal(doms_new[i], doms_cur[j]) && dattrs_equal(dattr_new, i, dattr_cur, j)) goto match2; @@ -8832,7 +7259,6 @@ void __init sched_init_smp(void) { cpumask_var_t non_isolated_cpus; - init_cpu_efficiency(); alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL); alloc_cpumask_var(&fallback_doms, GFP_KERNEL); @@ -8863,6 +7289,7 @@ void __init sched_init_smp(void) free_cpumask_var(non_isolated_cpus); init_sched_rt_class(); + init_sched_dl_class(); } #else void __init sched_init_smp(void) @@ -8896,11 +7323,6 @@ void __init sched_init(void) int i, j; unsigned long alloc_size = 0, ptr; - if (sched_enable_hmp) - pr_info("HMP scheduling enabled.\n"); - - BUG_ON(num_possible_cpus() > BITS_PER_LONG); - #ifdef CONFIG_FAIR_GROUP_SCHED alloc_size += 2 * nr_cpu_ids * sizeof(void **); #endif @@ -8937,13 +7359,15 @@ void __init sched_init(void) #endif /* CONFIG_CPUMASK_OFFSTACK */ } + init_rt_bandwidth(&def_rt_bandwidth, + global_rt_period(), global_rt_runtime()); + init_dl_bandwidth(&def_dl_bandwidth, + global_rt_period(), global_rt_runtime()); + #ifdef CONFIG_SMP init_defrootdomain(); #endif - init_rt_bandwidth(&def_rt_bandwidth, - global_rt_period(), global_rt_runtime()); - #ifdef CONFIG_RT_GROUP_SCHED init_rt_bandwidth(&root_task_group.rt_bandwidth, global_rt_period(), global_rt_runtime()); @@ -8967,6 +7391,7 @@ void __init sched_init(void) rq->calc_load_update = jiffies + LOAD_FREQ; init_cfs_rq(&rq->cfs); init_rt_rq(&rq->rt, rq); + init_dl_rq(&rq->dl, rq); #ifdef CONFIG_FAIR_GROUP_SCHED root_task_group.shares = ROOT_TASK_GROUP_LOAD; INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); @@ -8995,7 +7420,6 @@ void __init sched_init(void) rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; #ifdef CONFIG_RT_GROUP_SCHED - INIT_LIST_HEAD(&rq->leaf_rt_rq_list); init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); #endif @@ -9007,44 +7431,16 @@ void __init sched_init(void) #ifdef CONFIG_SMP rq->sd = NULL; rq->rd = NULL; - rq->cpu_power = SCHED_CAPACITY_SCALE; + rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE; rq->post_schedule = 0; rq->active_balance = 0; rq->next_balance = jiffies; rq->push_cpu = 0; - rq->push_task = NULL; rq->cpu = i; rq->online = 0; rq->idle_stamp = 0; rq->avg_idle = 2*sysctl_sched_migration_cost; - rq->cstate = 0; - rq->wakeup_latency = 0; - rq->wakeup_energy = 0; -#ifdef CONFIG_SCHED_HMP - rq->cur_freq = 1; - rq->max_freq = 1; - rq->min_freq = 1; - rq->max_possible_freq = 1; - rq->max_possible_capacity = 0; - rq->hmp_stats.cumulative_runnable_avg = 0; - rq->efficiency = 1024; - rq->capacity = 1024; - rq->load_scale_factor = 1024; - rq->window_start = 0; - rq->hmp_stats.nr_small_tasks = rq->hmp_stats.nr_big_tasks = 0; - rq->hmp_flags = 0; - rq->mostly_idle_load = pct_to_real(20); - rq->mostly_idle_nr_run = 3; - rq->mostly_idle_freq = 0; - rq->cur_irqload = 0; - rq->avg_irqload = 0; - rq->irqload_ts = 0; - rq->prefer_idle = 1; -#ifdef CONFIG_SCHED_FREQ_INPUT - rq->old_busy_time = 0; - rq->curr_runnable_sum = rq->prev_runnable_sum = 0; -#endif -#endif + rq->max_idle_balance_cost = sysctl_sched_migration_cost; INIT_LIST_HEAD(&rq->cfs_tasks); @@ -9060,18 +7456,12 @@ void __init sched_init(void) atomic_set(&rq->nr_iowait, 0); } - set_hmp_defaults(); - set_load_weight(&init_task); #ifdef CONFIG_PREEMPT_NOTIFIERS INIT_HLIST_HEAD(&init_task.preempt_notifiers); #endif -#ifdef CONFIG_RT_MUTEXES - plist_head_init(&init_task.pi_waiters); -#endif - /* * The boot idle thread does lazy MMU switching as well: */ @@ -9099,6 +7489,7 @@ void __init sched_init(void) if (cpu_isolated_map == NULL) zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT); idle_thread_set_boot_cpu(); + set_cpu_rq_start_time(); #endif init_sched_fair_class(); @@ -9126,8 +7517,8 @@ void __might_sleep(const char *file, int line, int preempt_offset) static unsigned long prev_jiffy; /* ratelimiting */ rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */ - if ((preempt_count_equals(preempt_offset) && !irqs_disabled()) || - oops_in_progress) + if ((preempt_count_equals(preempt_offset) && !irqs_disabled() && + !is_idle_task(current)) || oops_in_progress) return; if (system_state != SYSTEM_RUNNING && (!__might_sleep_init_called || system_state != SYSTEM_BOOTING)) @@ -9147,6 +7538,13 @@ void __might_sleep(const char *file, int line, int preempt_offset) debug_show_held_locks(current); if (irqs_disabled()) print_irqtrace_events(current); +#ifdef CONFIG_DEBUG_PREEMPT + if (!preempt_count_equals(preempt_offset)) { + pr_err("Preemption disabled at:"); + print_ip_sym(current->preempt_disable_ip); + pr_cont("\n"); + } +#endif dump_stack(); } EXPORT_SYMBOL(__might_sleep); @@ -9160,15 +7558,15 @@ static void normalize_task(struct rq *rq, struct task_struct *p) .sched_policy = SCHED_NORMAL, }; int old_prio = p->prio; - int on_rq; + int queued; - on_rq = p->on_rq; - if (on_rq) + queued = task_on_rq_queued(p); + if (queued) dequeue_task(rq, p, 0); - __setscheduler(rq, p, &attr); - if (on_rq) { + __setscheduler(rq, p, &attr, false); + if (queued) { enqueue_task(rq, p, 0); - resched_task(rq->curr); + resched_curr(rq); } check_class_changed(rq, p, prev_class, old_prio); @@ -9180,12 +7578,12 @@ void normalize_rt_tasks(void) unsigned long flags; struct rq *rq; - read_lock_irqsave(&tasklist_lock, flags); - do_each_thread(g, p) { + read_lock(&tasklist_lock); + for_each_process_thread(g, p) { /* * Only normalize user tasks: */ - if (!p->mm) + if (p->flags & PF_KTHREAD) continue; p->se.exec_start = 0; @@ -9195,26 +7593,21 @@ void normalize_rt_tasks(void) p->se.statistics.block_start = 0; #endif - if (!rt_task(p)) { + if (!dl_task(p) && !rt_task(p)) { /* * Renice negative nice level userspace * tasks back to 0: */ - if (TASK_NICE(p) < 0 && p->mm) + if (task_nice(p) < 0) set_user_nice(p, 0); continue; } - raw_spin_lock(&p->pi_lock); - rq = __task_rq_lock(p); - + rq = task_rq_lock(p, &flags); normalize_task(rq, p); - - __task_rq_unlock(rq); - raw_spin_unlock(&p->pi_lock); - } while_each_thread(g, p); - - read_unlock_irqrestore(&tasklist_lock, flags); + task_rq_unlock(rq, p, &flags); + } + read_unlock(&tasklist_lock); } #endif /* CONFIG_MAGIC_SYSRQ */ @@ -9235,6 +7628,8 @@ void normalize_rt_tasks(void) * @cpu: the processor in question. * * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! + * + * Return: The current task for @cpu. */ struct task_struct *curr_task(int cpu) { @@ -9352,20 +7747,25 @@ void sched_offline_group(struct task_group *tg) void sched_move_task(struct task_struct *tsk) { struct task_group *tg; - int on_rq, running; + int queued, running; unsigned long flags; struct rq *rq; rq = task_rq_lock(tsk, &flags); running = task_current(rq, tsk); - on_rq = tsk->on_rq; + queued = task_on_rq_queued(tsk); - if (on_rq) + if (queued) dequeue_task(rq, tsk, 0); if (unlikely(running)) - tsk->sched_class->put_prev_task(rq, tsk); + put_prev_task(rq, tsk); + /* + * All callers are synchronized by task_rq_lock(); we do not use RCU + * which is pointless here. Thus, we pass "true" to task_css_check() + * to prevent lockdep warnings. + */ tg = container_of(task_subsys_state_check(tsk, cpu_cgroup_subsys_id, lockdep_is_held(&tsk->sighand->siglock)), struct task_group, css); @@ -9374,30 +7774,20 @@ void sched_move_task(struct task_struct *tsk) #ifdef CONFIG_FAIR_GROUP_SCHED if (tsk->sched_class->task_move_group) - tsk->sched_class->task_move_group(tsk, on_rq); + tsk->sched_class->task_move_group(tsk, queued); else #endif set_task_rq(tsk, task_cpu(tsk)); if (unlikely(running)) tsk->sched_class->set_curr_task(rq); - if (on_rq) + if (queued) enqueue_task(rq, tsk, 0); task_rq_unlock(rq, tsk, &flags); } #endif /* CONFIG_CGROUP_SCHED */ -#if defined(CONFIG_RT_GROUP_SCHED) || defined(CONFIG_CFS_BANDWIDTH) -static unsigned long to_ratio(u64 period, u64 runtime) -{ - if (runtime == RUNTIME_INF) - return 1ULL << 20; - - return div64_u64(runtime << 20, period); -} -#endif - #ifdef CONFIG_RT_GROUP_SCHED /* * Ensure that the real time constraints are schedulable. @@ -9409,10 +7799,16 @@ static inline int tg_has_rt_tasks(struct task_group *tg) { struct task_struct *g, *p; - do_each_thread(g, p) { - if (rt_task(p) && task_rq(p)->rt.tg == tg) + /* + * Autogroups do not have RT tasks; see autogroup_create(). + */ + if (task_group_is_autogroup(tg)) + return 0; + + for_each_process_thread(g, p) { + if (rt_task(p) && task_group(p) == tg) return 1; - } while_each_thread(g, p); + } return 0; } @@ -9571,23 +7967,12 @@ static long sched_group_rt_period(struct task_group *tg) do_div(rt_period_us, NSEC_PER_USEC); return rt_period_us; } +#endif /* CONFIG_RT_GROUP_SCHED */ -static int sched_rt_global_constraints(void) -{ - u64 runtime, period; - int ret = 0; - - if (sysctl_sched_rt_period <= 0) - return -EINVAL; - - runtime = global_rt_runtime(); - period = global_rt_period(); - - /* - * Sanity check on the sysctl variables. - */ - if (runtime > period && runtime != RUNTIME_INF) - return -EINVAL; +#ifdef CONFIG_RT_GROUP_SCHED +static int sched_rt_global_constraints(void) +{ + int ret = 0; mutex_lock(&rt_constraints_mutex); read_lock(&tasklist_lock); @@ -9611,17 +7996,7 @@ static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk) static int sched_rt_global_constraints(void) { unsigned long flags; - int i; - - if (sysctl_sched_rt_period <= 0) - return -EINVAL; - - /* - * There's always some RT tasks in the root group - * -- migration, kstopmachine etc.. - */ - if (sysctl_sched_rt_runtime == 0) - return -EBUSY; + int i, ret = 0; raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); for_each_possible_cpu(i) { @@ -9633,36 +8008,99 @@ static int sched_rt_global_constraints(void) } raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); - return 0; + return ret; } #endif /* CONFIG_RT_GROUP_SCHED */ -int sched_rr_handler(struct ctl_table *table, int write, - void __user *buffer, size_t *lenp, - loff_t *ppos) +static int sched_dl_global_constraints(void) { - int ret; - static DEFINE_MUTEX(mutex); + u64 runtime = global_rt_runtime(); + u64 period = global_rt_period(); + u64 new_bw = to_ratio(period, runtime); + struct dl_bw *dl_b; + int cpu, ret = 0; + unsigned long flags; - mutex_lock(&mutex); - ret = proc_dointvec(table, write, buffer, lenp, ppos); - /* make sure that internally we keep jiffies */ - /* also, writing zero resets timeslice to default */ - if (!ret && write) { - sched_rr_timeslice = sched_rr_timeslice <= 0 ? - RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice); + /* + * Here we want to check the bandwidth not being set to some + * value smaller than the currently allocated bandwidth in + * any of the root_domains. + * + * FIXME: Cycling on all the CPUs is overdoing, but simpler than + * cycling on root_domains... Discussion on different/better + * solutions is welcome! + */ + for_each_possible_cpu(cpu) { + rcu_read_lock_sched(); + dl_b = dl_bw_of(cpu); + + raw_spin_lock_irqsave(&dl_b->lock, flags); + if (new_bw < dl_b->total_bw) + ret = -EBUSY; + raw_spin_unlock_irqrestore(&dl_b->lock, flags); + + rcu_read_unlock_sched(); + + if (ret) + break; } - mutex_unlock(&mutex); + return ret; } +static void sched_dl_do_global(void) +{ + u64 new_bw = -1; + struct dl_bw *dl_b; + int cpu; + unsigned long flags; + + def_dl_bandwidth.dl_period = global_rt_period(); + def_dl_bandwidth.dl_runtime = global_rt_runtime(); + + if (global_rt_runtime() != RUNTIME_INF) + new_bw = to_ratio(global_rt_period(), global_rt_runtime()); + + /* + * FIXME: As above... + */ + for_each_possible_cpu(cpu) { + rcu_read_lock_sched(); + dl_b = dl_bw_of(cpu); + + raw_spin_lock_irqsave(&dl_b->lock, flags); + dl_b->bw = new_bw; + raw_spin_unlock_irqrestore(&dl_b->lock, flags); + + rcu_read_unlock_sched(); + } +} + +static int sched_rt_global_validate(void) +{ + if (sysctl_sched_rt_period <= 0) + return -EINVAL; + + if ((sysctl_sched_rt_runtime != RUNTIME_INF) && + (sysctl_sched_rt_runtime > sysctl_sched_rt_period)) + return -EINVAL; + + return 0; +} + +static void sched_rt_do_global(void) +{ + def_rt_bandwidth.rt_runtime = global_rt_runtime(); + def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period()); +} + int sched_rt_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { - int ret; int old_period, old_runtime; static DEFINE_MUTEX(mutex); + int ret; mutex_lock(&mutex); old_period = sysctl_sched_rt_period; @@ -9671,21 +8109,50 @@ int sched_rt_handler(struct ctl_table *table, int write, ret = proc_dointvec(table, write, buffer, lenp, ppos); if (!ret && write) { + ret = sched_rt_global_validate(); + if (ret) + goto undo; + ret = sched_rt_global_constraints(); - if (ret) { - sysctl_sched_rt_period = old_period; - sysctl_sched_rt_runtime = old_runtime; - } else { - def_rt_bandwidth.rt_runtime = global_rt_runtime(); - def_rt_bandwidth.rt_period = - ns_to_ktime(global_rt_period()); - } + if (ret) + goto undo; + + ret = sched_dl_global_constraints(); + if (ret) + goto undo; + + sched_rt_do_global(); + sched_dl_do_global(); + } + if (0) { +undo: + sysctl_sched_rt_period = old_period; + sysctl_sched_rt_runtime = old_runtime; } mutex_unlock(&mutex); return ret; } +int sched_rr_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + static DEFINE_MUTEX(mutex); + + mutex_lock(&mutex); + ret = proc_dointvec(table, write, buffer, lenp, ppos); + /* make sure that internally we keep jiffies */ + /* also, writing zero resets timeslice to default */ + if (!ret && write) { + sched_rr_timeslice = sched_rr_timeslice <= 0 ? + RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice); + } + mutex_unlock(&mutex); + return ret; +} + #ifdef CONFIG_CGROUP_SCHED /* return corresponding task_group object of a cgroup */ @@ -9781,6 +8248,7 @@ cpu_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp, sched_move_task(task); } +#ifdef TJK_HMP static u64 cpu_notify_on_migrate_read_u64(struct cgroup *cgrp, struct cftype *cft) { @@ -9798,6 +8266,7 @@ static int cpu_notify_on_migrate_write_u64(struct cgroup *cgrp, return 0; } +#endif #ifdef CONFIG_SCHED_HMP @@ -10116,11 +8585,13 @@ static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft) #endif /* CONFIG_RT_GROUP_SCHED */ static struct cftype cpu_files[] = { +#ifdef TJK_HMP { .name = "notify_on_migrate", .read_u64 = cpu_notify_on_migrate_read_u64, .write_u64 = cpu_notify_on_migrate_write_u64, }, +#endif #ifdef CONFIG_SCHED_HMP { .name = "upmigrate_discourage", @@ -10189,3 +8660,179 @@ void dump_cpu_task(int cpu) sched_show_task(cpu_curr(cpu)); } +// TJK: Below here: Added for 3.10 compatibility +static long __sched +sleep_on_common(wait_queue_head_t *q, int state, long timeout) +{ + unsigned long flags; + wait_queue_t wait; + + init_waitqueue_entry(&wait, current); + + __set_current_state(state); + + spin_lock_irqsave(&q->lock, flags); + __add_wait_queue(q, &wait); + spin_unlock(&q->lock); + timeout = schedule_timeout(timeout); + spin_lock_irq(&q->lock); + __remove_wait_queue(q, &wait); + spin_unlock_irqrestore(&q->lock, flags); + + return timeout; +} + +void __sched interruptible_sleep_on(wait_queue_head_t *q) +{ + sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); +} +EXPORT_SYMBOL(interruptible_sleep_on); + +long __sched +interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout) +{ + return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout); +} +EXPORT_SYMBOL(interruptible_sleep_on_timeout); + +void __sched sleep_on(wait_queue_head_t *q) +{ + sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); +} +EXPORT_SYMBOL(sleep_on); + +/* + * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just + * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve + * number) then we wake all the non-exclusive tasks and one exclusive task. + * + * There are circumstances in which we can try to wake a task which has already + * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns + * zero in this (rare) case, and we handle it by continuing to scan the queue. + */ +static void __wake_up_common(wait_queue_head_t *q, unsigned int mode, + int nr_exclusive, int wake_flags, void *key) +{ + wait_queue_t *curr, *next; + + list_for_each_entry_safe(curr, next, &q->task_list, task_list) { + unsigned flags = curr->flags; + + if (curr->func(curr, mode, wake_flags, key) && + (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive) + break; + } +} + +/** + * __wake_up - wake up threads blocked on a waitqueue. + * @q: the waitqueue + * @mode: which threads + * @nr_exclusive: how many wake-one or wake-many threads to wake up + * @key: is directly passed to the wakeup function + * + * It may be assumed that this function implies a write memory barrier before + * changing the task state if and only if any tasks are woken up. + */ +void __wake_up(wait_queue_head_t *q, unsigned int mode, + int nr_exclusive, void *key) +{ + unsigned long flags; + + spin_lock_irqsave(&q->lock, flags); + __wake_up_common(q, mode, nr_exclusive, 0, key); + spin_unlock_irqrestore(&q->lock, flags); +} +EXPORT_SYMBOL(__wake_up); + +/* + * Same as __wake_up but called with the spinlock in wait_queue_head_t held. + */ +void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr) +{ + __wake_up_common(q, mode, nr, 0, NULL); +} +EXPORT_SYMBOL_GPL(__wake_up_locked); + +void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key) +{ + __wake_up_common(q, mode, 1, 0, key); +} +EXPORT_SYMBOL_GPL(__wake_up_locked_key); + +/** + * __wake_up_sync_key - wake up threads blocked on a waitqueue. + * @q: the waitqueue + * @mode: which threads + * @nr_exclusive: how many wake-one or wake-many threads to wake up + * @key: opaque value to be passed to wakeup targets + * + * The sync wakeup differs that the waker knows that it will schedule + * away soon, so while the target thread will be woken up, it will not + * be migrated to another CPU - ie. the two threads are 'synchronized' + * with each other. This can prevent needless bouncing between CPUs. + * + * On UP it can prevent extra preemption. + * + * It may be assumed that this function implies a write memory barrier before + * changing the task state if and only if any tasks are woken up. + */ +void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, + int nr_exclusive, void *key) +{ + unsigned long flags; + int wake_flags = WF_SYNC; + + if (unlikely(!q)) + return; + + if (unlikely(!nr_exclusive)) + wake_flags = 0; + + spin_lock_irqsave(&q->lock, flags); + __wake_up_common(q, mode, nr_exclusive, wake_flags, key); + spin_unlock_irqrestore(&q->lock, flags); +} +EXPORT_SYMBOL_GPL(__wake_up_sync_key); + +/* + * __wake_up_sync - see __wake_up_sync_key() + */ +void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive) +{ + __wake_up_sync_key(q, mode, nr_exclusive, NULL); +} +EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ + +/* + * Note C-state for (idle) cpus. + * + * @cstate = cstate index, 0 -> active state + * @wakeup_energy = energy spent in waking up cpu + * @wakeup_latency = latency to wakeup from cstate + * + */ +void +sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency) +{ + // TJK: Need to integrate QC cpuidle with EAS (doesn't use standard + // cpuidle framework) +#if 0 + struct rq *rq = cpu_rq(cpu); + + rq->cstate = cstate; /* C1, C2 etc */ + rq->wakeup_energy = wakeup_energy; + rq->wakeup_latency = wakeup_latency; +#else + if ((cstate == 0 && wakeup_energy == 0 && wakeup_latency == 0) || + (cstate == 1 && cpu < 4 && wakeup_energy == 5800 && wakeup_latency == 50) || + (cstate == 1 && cpu >= 4 && wakeup_energy == 24120 && wakeup_latency == 60) || + (cstate == 3 && cpu >= 4 && wakeup_energy == 155775 && wakeup_latency == 475) || + (cstate == 4 && cpu < 4 && wakeup_energy == 46400 && wakeup_latency == 480) || + (cstate == 4 && cpu >= 4 && wakeup_energy == 192156 && wakeup_latency == 636)) { + } else { + pr_info("TJK: CPU%d ->cstate=%d nrg=%d latency=%d\n", cpu, cstate, wakeup_energy, wakeup_latency); + } +#endif +} + diff --git a/kernel/sched/cpuacct.c b/kernel/sched/cpuacct.c index dbb7e2cd95eba..f0186c764ee14 100644 --- a/kernel/sched/cpuacct.c +++ b/kernel/sched/cpuacct.c @@ -33,7 +33,11 @@ struct cpuacct { struct kernel_cpustat __percpu *cpustat; }; -/* return cpu accounting group corresponding to this container */ +static inline struct cpuacct *css_ca(struct cgroup_subsys_state *css) +{ + return css ? container_of(css, struct cpuacct, css) : NULL; +} + static inline struct cpuacct *cgroup_ca(struct cgroup *cgrp) { return container_of(cgroup_subsys_state(cgrp, cpuacct_subsys_id), @@ -43,8 +47,7 @@ static inline struct cpuacct *cgroup_ca(struct cgroup *cgrp) /* return cpu accounting group to which this task belongs */ static inline struct cpuacct *task_ca(struct task_struct *tsk) { - return container_of(task_subsys_state(tsk, cpuacct_subsys_id), - struct cpuacct, css); + return container_of(task_subsys_state(tsk, cpuacct_subsys_id), struct cpuacct, css); } static inline struct cpuacct *__parent_ca(struct cpuacct *ca) @@ -172,8 +175,7 @@ static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype, return err; } -static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft, - struct seq_file *m) +static int cpuacct_percpu_seq_show(struct cgroup *cgroup, struct cftype *cft, struct seq_file *m) { struct cpuacct *ca = cgroup_ca(cgroup); u64 percpu; @@ -229,7 +231,7 @@ static struct cftype files[] = { }, { .name = "usage_percpu", - .read_seq_string = cpuacct_percpu_seq_read, + .read_seq_string = cpuacct_percpu_seq_show, }, { .name = "stat", @@ -281,16 +283,16 @@ void cpuacct_account_field(struct task_struct *p, int index, u64 val) while (ca != &root_cpuacct) { kcpustat = this_cpu_ptr(ca->cpustat); kcpustat->cpustat[index] += val; - ca = __parent_ca(ca); + ca = parent_ca(ca); } rcu_read_unlock(); } struct cgroup_subsys cpuacct_subsys = { - .name = "cpuacct", + .name = "cpuacct", .css_alloc = cpuacct_css_alloc, .css_free = cpuacct_css_free, - .subsys_id = cpuacct_subsys_id, + .subsys_id = cpuacct_subsys_id, .base_cftypes = files, .early_init = 1, }; diff --git a/kernel/sched/cpudeadline.c b/kernel/sched/cpudeadline.c new file mode 100644 index 0000000000000..539ca3ce071b2 --- /dev/null +++ b/kernel/sched/cpudeadline.c @@ -0,0 +1,227 @@ +/* + * kernel/sched/cpudl.c + * + * Global CPU deadline management + * + * Author: Juri Lelli + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + */ + +#include +#include +#include +#include "cpudeadline.h" + +static inline int parent(int i) +{ + return (i - 1) >> 1; +} + +static inline int left_child(int i) +{ + return (i << 1) + 1; +} + +static inline int right_child(int i) +{ + return (i << 1) + 2; +} + +static inline int dl_time_before(u64 a, u64 b) +{ + return (s64)(a - b) < 0; +} + +static void cpudl_exchange(struct cpudl *cp, int a, int b) +{ + int cpu_a = cp->elements[a].cpu, cpu_b = cp->elements[b].cpu; + + swap(cp->elements[a].cpu, cp->elements[b].cpu); + swap(cp->elements[a].dl , cp->elements[b].dl ); + + swap(cp->elements[cpu_a].idx, cp->elements[cpu_b].idx); +} + +static void cpudl_heapify(struct cpudl *cp, int idx) +{ + int l, r, largest; + + /* adapted from lib/prio_heap.c */ + while(1) { + l = left_child(idx); + r = right_child(idx); + largest = idx; + + if ((l < cp->size) && dl_time_before(cp->elements[idx].dl, + cp->elements[l].dl)) + largest = l; + if ((r < cp->size) && dl_time_before(cp->elements[largest].dl, + cp->elements[r].dl)) + largest = r; + if (largest == idx) + break; + + /* Push idx down the heap one level and bump one up */ + cpudl_exchange(cp, largest, idx); + idx = largest; + } +} + +static void cpudl_change_key(struct cpudl *cp, int idx, u64 new_dl) +{ + WARN_ON(idx == IDX_INVALID || !cpu_present(idx)); + + if (dl_time_before(new_dl, cp->elements[idx].dl)) { + cp->elements[idx].dl = new_dl; + cpudl_heapify(cp, idx); + } else { + cp->elements[idx].dl = new_dl; + while (idx > 0 && dl_time_before(cp->elements[parent(idx)].dl, + cp->elements[idx].dl)) { + cpudl_exchange(cp, idx, parent(idx)); + idx = parent(idx); + } + } +} + +static inline int cpudl_maximum(struct cpudl *cp) +{ + return cp->elements[0].cpu; +} + +/* + * cpudl_find - find the best (later-dl) CPU in the system + * @cp: the cpudl max-heap context + * @p: the task + * @later_mask: a mask to fill in with the selected CPUs (or NULL) + * + * Returns: int - best CPU (heap maximum if suitable) + */ +int cpudl_find(struct cpudl *cp, struct task_struct *p, + struct cpumask *later_mask) +{ + int best_cpu = -1; + const struct sched_dl_entity *dl_se = &p->dl; + + if (later_mask && cpumask_and(later_mask, later_mask, cp->free_cpus)) { + best_cpu = cpumask_any(later_mask); + goto out; + } else if (cpumask_test_cpu(cpudl_maximum(cp), &p->cpus_allowed) && + dl_time_before(dl_se->deadline, cp->elements[0].dl)) { + best_cpu = cpudl_maximum(cp); + if (later_mask) + cpumask_set_cpu(best_cpu, later_mask); + } + +out: + WARN_ON(best_cpu != -1 && !cpu_present(best_cpu)); + + return best_cpu; +} + +/* + * cpudl_set - update the cpudl max-heap + * @cp: the cpudl max-heap context + * @cpu: the target cpu + * @dl: the new earliest deadline for this cpu + * + * Notes: assumes cpu_rq(cpu)->lock is locked + * + * Returns: (void) + */ +void cpudl_set(struct cpudl *cp, int cpu, u64 dl, int is_valid) +{ + int old_idx, new_cpu; + unsigned long flags; + + WARN_ON(!cpu_present(cpu)); + + raw_spin_lock_irqsave(&cp->lock, flags); + old_idx = cp->elements[cpu].idx; + if (!is_valid) { + /* remove item */ + if (old_idx == IDX_INVALID) { + /* + * Nothing to remove if old_idx was invalid. + * This could happen if a rq_offline_dl is + * called for a CPU without -dl tasks running. + */ + goto out; + } + new_cpu = cp->elements[cp->size - 1].cpu; + cp->elements[old_idx].dl = cp->elements[cp->size - 1].dl; + cp->elements[old_idx].cpu = new_cpu; + cp->size--; + cp->elements[new_cpu].idx = old_idx; + cp->elements[cpu].idx = IDX_INVALID; + while (old_idx > 0 && dl_time_before( + cp->elements[parent(old_idx)].dl, + cp->elements[old_idx].dl)) { + cpudl_exchange(cp, old_idx, parent(old_idx)); + old_idx = parent(old_idx); + } + cpumask_set_cpu(cpu, cp->free_cpus); + cpudl_heapify(cp, old_idx); + + goto out; + } + + if (old_idx == IDX_INVALID) { + cp->size++; + cp->elements[cp->size - 1].dl = 0; + cp->elements[cp->size - 1].cpu = cpu; + cp->elements[cpu].idx = cp->size - 1; + cpudl_change_key(cp, cp->size - 1, dl); + cpumask_clear_cpu(cpu, cp->free_cpus); + } else { + cpudl_change_key(cp, old_idx, dl); + } + +out: + raw_spin_unlock_irqrestore(&cp->lock, flags); +} + +/* + * cpudl_init - initialize the cpudl structure + * @cp: the cpudl max-heap context + */ +int cpudl_init(struct cpudl *cp) +{ + int i; + + memset(cp, 0, sizeof(*cp)); + raw_spin_lock_init(&cp->lock); + cp->size = 0; + + cp->elements = kcalloc(nr_cpu_ids, + sizeof(struct cpudl_item), + GFP_KERNEL); + if (!cp->elements) + return -ENOMEM; + + if (!alloc_cpumask_var(&cp->free_cpus, GFP_KERNEL)) { + kfree(cp->elements); + return -ENOMEM; + } + + for_each_possible_cpu(i) + cp->elements[i].idx = IDX_INVALID; + + cpumask_setall(cp->free_cpus); + + return 0; +} + +/* + * cpudl_cleanup - clean up the cpudl structure + * @cp: the cpudl max-heap context + */ +void cpudl_cleanup(struct cpudl *cp) +{ + free_cpumask_var(cp->free_cpus); + kfree(cp->elements); +} diff --git a/kernel/sched/cpudeadline.h b/kernel/sched/cpudeadline.h new file mode 100644 index 0000000000000..538c9796ad4a0 --- /dev/null +++ b/kernel/sched/cpudeadline.h @@ -0,0 +1,33 @@ +#ifndef _LINUX_CPUDL_H +#define _LINUX_CPUDL_H + +#include + +#define IDX_INVALID -1 + +struct cpudl_item { + u64 dl; + int cpu; + int idx; +}; + +struct cpudl { + raw_spinlock_t lock; + int size; + cpumask_var_t free_cpus; + struct cpudl_item *elements; +}; + + +#ifdef CONFIG_SMP +int cpudl_find(struct cpudl *cp, struct task_struct *p, + struct cpumask *later_mask); +void cpudl_set(struct cpudl *cp, int cpu, u64 dl, int is_valid); +int cpudl_init(struct cpudl *cp); +void cpudl_cleanup(struct cpudl *cp); +#else +#define cpudl_set(cp, cpu, dl) do { } while (0) +#define cpudl_init() do { } while (0) +#endif /* CONFIG_SMP */ + +#endif /* _LINUX_CPUDL_H */ diff --git a/kernel/sched/cpufreq_sched.c b/kernel/sched/cpufreq_sched.c new file mode 100644 index 0000000000000..5f0b3fe15db67 --- /dev/null +++ b/kernel/sched/cpufreq_sched.c @@ -0,0 +1,464 @@ +/* + * Copyright (C) 2015 Michael Turquette + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include + +#include "sched.h" + +#define THROTTLE_NSEC 20000000 /* 50ms default */ + +static DEFINE_PER_CPU(unsigned long, pcpu_capacity); +static DEFINE_PER_CPU(struct cpufreq_policy *, pcpu_policy); +static DEFINE_PER_CPU(int, governor_started); + +/** + * gov_data - per-policy data internal to the governor + * @throttle: next throttling period expiry. Derived from throttle_nsec + * @throttle_nsec: throttle period length in nanoseconds + * @freq: new frequency stored in *_sched_update_cpu and used in *_sched_thread + * + * struct gov_data is the per-policy cpufreq_sched-specific data structure. A + * per-policy instance of it is created when the cpufreq_sched governor receives + * the CPUFREQ_GOV_START condition and a pointer to it exists in the gov_data + * member of struct cpufreq_policy. + * + * Readers of this data must call down_read(policy->rwsem). Writers must + * call down_write(policy->rwsem). + */ +struct gov_data { + ktime_t throttle; + unsigned int throttle_nsec; + struct cpufreq_policy *policy; + unsigned int freq; + bool change_pending; + struct list_head gov_list; +}; + + /* worker thread for dvfs transition that may block/sleep */ +static struct task_struct *freq_change_task; +static LIST_HEAD(sched_gov_list); +static struct mutex gov_list_lock; + +/* irq_work: callback used to wake up worker thread */ +static struct irq_work irq_work; + +static void cpufreq_sched_try_driver_target(struct cpufreq_policy *policy, unsigned int freq) +{ + struct gov_data *gd; + + /* avoid race with cpufreq_sched_stop */ + if (!down_write_trylock(&policy->rwsem)) + return; + + gd = policy->governor_data; + + if (!gd) + return; + + __cpufreq_driver_target(policy, freq, CPUFREQ_RELATION_L); + + gd->throttle = ktime_add_ns(ktime_get(), gd->throttle_nsec); + up_write(&policy->rwsem); +} + +/* + * we pass in struct cpufreq_policy. This is safe because changing out the + * policy requires a call to __cpufreq_governor(policy, CPUFREQ_GOV_STOP), + * which tears down all of the data structures and __cpufreq_governor(policy, + * CPUFREQ_GOV_START) will do a full rebuild, including this kthread with the + * new policy pointer + */ +static int cpufreq_sched_thread(void *data) +{ + struct sched_param param; + int ret; + + param.sched_priority = 50; + ret = sched_setscheduler_nocheck(current, SCHED_FIFO, ¶m); + if (ret) { + pr_warn("%s: failed to set SCHED_FIFO\n", __func__); + do_exit(-EINVAL); + } else { + pr_debug("%s: kthread (%d) set to SCHED_FIFO\n", + __func__, current->pid); + } + + /* main loop of the frequency change kthread */ + do { + struct cpufreq_policy *policy = NULL; + unsigned int freq = 0; + struct gov_data *gd = NULL; + mutex_lock(&gov_list_lock); + list_for_each_entry(gd, &sched_gov_list, gov_list) { + if (gd->change_pending) { + gd->change_pending = false; + freq = gd->freq; + policy = gd->policy; + break; + } + } + mutex_unlock(&gov_list_lock); + + if (!policy) { + set_current_state(TASK_INTERRUPTIBLE); + schedule(); + } else { + cpufreq_sched_try_driver_target(policy, freq); + } + + if (kthread_should_stop()) + break; + + } while (!kthread_should_stop()); + + do_exit(0); +} + +static void cpufreq_sched_irq_work(struct irq_work *irq_work) +{ + wake_up_process(freq_change_task); +} + +/** + * cpufreq_sched_set_capacity - interface to scheduler for changing capacity values + * @cpu: cpu whose capacity utilization has recently changed + * @capacity: the new capacity requested by cpu + * + * cpufreq_sched_sched_capacity is an interface exposed to the scheduler so + * that the scheduler may inform the governor of updates to capacity + * utilization and make changes to cpu frequency. Currently this interface is + * designed around PELT values in CFS. It can be expanded to other scheduling + * classes in the future if needed. + * + * cpufreq_sched_set_capacity raises an IPI. The irq_work handler for that IPI + * wakes up the thread that does the actual work, cpufreq_sched_thread. + * + * This functions bails out early if either condition is true: + * 1) this cpu did not the new maximum capacity for its frequency domain + * 2) no change in cpu frequency is necessary to meet the new capacity request + */ +void cpufreq_sched_set_cap(int cpu, unsigned long capacity) +{ + unsigned int freq_new, cpu_tmp; + struct cpufreq_policy *policy; + struct gov_data *gd; + unsigned long capacity_max = 0; + + if (!per_cpu(governor_started, cpu)) + return; + + /* update per-cpu capacity request */ + per_cpu(pcpu_capacity, cpu) = capacity; + + policy = cpufreq_cpu_get(cpu); + if (IS_ERR_OR_NULL(policy)) { + return; + } + + if (!policy->governor_data) + goto out; + + gd = policy->governor_data; + + /* bail early if we are throttled */ + if (ktime_compare(ktime_get(), gd->throttle) < 0) + goto out; + + /* find max capacity requested by cpus in this policy */ + for_each_cpu(cpu_tmp, policy->cpus) + capacity_max = max(capacity_max, per_cpu(pcpu_capacity, cpu_tmp)); + + /* + * We only change frequency if this cpu's capacity request represents a + * new max. If another cpu has requested a capacity greater than the + * previous max then we rely on that cpu to hit this code path and make + * the change. IOW, the cpu with the new max capacity is responsible + * for setting the new capacity/frequency. + * + * If this cpu is not the new maximum then bail + */ + if (capacity_max > capacity) + goto out; + + /* Convert the new maximum capacity request into a cpu frequency */ + freq_new = (capacity * policy->max) / capacity_orig_of(cpu); + + /* No change in frequency? Bail and return current capacity. */ + if (freq_new == policy->cur) + goto out; + + /* store the new frequency and perform the transition */ + gd->freq = freq_new; + gd->change_pending = true; + + irq_work_queue(&irq_work); + +out: + cpufreq_cpu_put(policy); + return; +} + +/** + * cpufreq_sched_reset_capacity - interface to scheduler for resetting capacity + * requests + * @cpu: cpu whose capacity request has to be reset + * + * This _wont trigger_ any capacity update. + */ +void cpufreq_sched_reset_cap(int cpu) +{ + per_cpu(pcpu_capacity, cpu) = 0; +} + +static inline void set_sched_energy_freq(void) +{ + if (!sched_energy_freq()) + static_key_slow_inc(&__sched_energy_freq); +} + +static inline void clear_sched_energy_freq(void) +{ + if (sched_energy_freq()) + static_key_slow_dec(&__sched_energy_freq); +} + +static int cpufreq_sched_policy_start(struct cpufreq_policy *policy) +{ + int cpu; + + /* initialize per-cpu data */ + for_each_cpu(cpu, policy->cpus) { + pr_info("%s: start CPU%d\n", __func__, cpu); + per_cpu(pcpu_capacity, cpu) = 0; + per_cpu(pcpu_policy, cpu) = policy; + } + return 0; +} + +static int cpufreq_sched_policy_stop(struct cpufreq_policy *policy) +{ + int cpu; + if (!policy) { + return 0; + } + for_each_cpu(cpu, policy->cpus) { + pr_info("%s: stop CPU%d\n", __func__, cpu); + } + /* + * Nothing to do. The per_cpu fields will be re-initialized + * on the next START + */ + return 0; +} + +static struct attribute_group sched_attr_group_gov_pol; +static struct attribute_group *get_sysfs_attr(void) +{ + return &sched_attr_group_gov_pol; +} + +static int cpufreq_sched_policy_init(struct cpufreq_policy *policy) +{ + int rc; + struct gov_data *gd; + int cpu = cpumask_first(policy->cpus); + + if (cpu >= nr_cpu_ids) { + return -EINVAL; + } + WARN_ON(policy->governor_data); + /* prepare per-policy private data */ + gd = kzalloc(sizeof(*gd), GFP_KERNEL); + pr_info("%s: init CPU%d cluster\n", __func__, cpu); + + if (!gd) { + pr_debug("%s: failed to allocate private data\n", __func__); + return -ENOMEM; + } + rc = sysfs_create_group(get_governor_parent_kobj(policy), get_sysfs_attr()); + if (rc) { + pr_err("%s: couldn't create sysfs attributes: %d\n", __func__, rc); + goto err; + } + + /* + * Don't ask for freq changes at an higher rate than what + * the driver advertises as transition latency. + */ +#ifdef THROTTLE_FROM_CPUFREQ_DRIVER_LATENCY + gd->throttle_nsec = policy->cpuinfo.transition_latency ? + policy->cpuinfo.transition_latency : + THROTTLE_NSEC; +#else + gd->throttle_nsec = THROTTLE_NSEC; +#endif + pr_debug("%s: throttle threshold = %u [ns]\n", + __func__, gd->throttle_nsec); + + policy->governor_data = gd; + gd->policy = policy; + + mutex_lock(&gov_list_lock); + list_add_tail(&gd->gov_list, &sched_gov_list); + mutex_unlock(&gov_list_lock); + + set_sched_energy_freq(); + + for_each_cpu(cpu, policy->cpus) + per_cpu(governor_started, cpu) = 1; + + return 0; + +err: + kfree(gd); + return -ENOMEM; +} + +static int cpufreq_sched_policy_exit(struct cpufreq_policy *policy) +{ + struct gov_data *gd = policy->governor_data; + int cpu = cpumask_first(policy->cpus); + pr_info("%s: exit CPU%d cluster\n", __func__, cpu); + WARN_ON(!policy->governor_data); + + for_each_cpu(cpu, policy->cpus) + per_cpu(governor_started, cpu) = 0; + + clear_sched_energy_freq(); + + policy->governor_data = NULL; + mutex_lock(&gov_list_lock); + list_del(&gd->gov_list); + mutex_unlock(&gov_list_lock); + + sysfs_remove_group(get_governor_parent_kobj(policy), get_sysfs_attr()); + + /* FIXME replace with devm counterparts? */ + kfree(gd); + return 0; +} + +static int cpufreq_sched_setup(struct cpufreq_policy *policy, unsigned int event) +{ + switch (event) { + case CPUFREQ_GOV_START: + return cpufreq_sched_policy_start(policy); + case CPUFREQ_GOV_STOP: + return cpufreq_sched_policy_stop(policy); + case CPUFREQ_GOV_POLICY_INIT: + return cpufreq_sched_policy_init(policy); + case CPUFREQ_GOV_POLICY_EXIT: + return cpufreq_sched_policy_exit(policy); + case CPUFREQ_GOV_LIMITS: /* unused */ + break; + } + return 0; +} + +/* Tunables */ +static ssize_t show_throttle_ns(struct gov_data *gd, char *buf) +{ + return sprintf(buf, "%u\n", gd->throttle_nsec); +} + +static ssize_t store_throttle_ns(struct gov_data *gd, + const char *buf, size_t count) +{ + int ret; + long unsigned int val; + + ret = kstrtoul(buf, 0, &val); + if (ret < 0) + return ret; + gd->throttle_nsec = val; + return count; +} +/* + * Create show/store routines + * - sys: One governor instance for complete SYSTEM + * - pol: One governor instance per struct cpufreq_policy + */ +#define show_gov_pol_sys(file_name) \ +static ssize_t show_##file_name##_gov_pol \ +(struct cpufreq_policy *policy, char *buf) \ +{ \ + return show_##file_name(policy->governor_data, buf); \ +} + +#define store_gov_pol_sys(file_name) \ +static ssize_t store_##file_name##_gov_pol \ +(struct cpufreq_policy *policy, const char *buf, size_t count) \ +{ \ + return store_##file_name(policy->governor_data, buf, count); \ +} + +#define gov_pol_attr_rw(_name) \ + static struct freq_attr _name##_gov_pol = \ + __ATTR(_name, 0644, show_##_name##_gov_pol, store_##_name##_gov_pol) + +#define show_store_gov_pol_sys(file_name) \ + show_gov_pol_sys(file_name); \ + store_gov_pol_sys(file_name) +#define tunable_handlers(file_name) \ + show_gov_pol_sys(file_name); \ + store_gov_pol_sys(file_name); \ + gov_pol_attr_rw(file_name) + +tunable_handlers(throttle_ns); + +/* Per policy governor instance */ +static struct attribute *sched_attributes_gov_pol[] = { + &throttle_ns_gov_pol.attr, + NULL, +}; + +static struct attribute_group sched_attr_group_gov_pol = { + .attrs = sched_attributes_gov_pol, + .name = "sched", +}; + +#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHED +static +#endif +struct cpufreq_governor cpufreq_gov_sched = { + .name = "sched", + .governor = cpufreq_sched_setup, + .owner = THIS_MODULE, +}; + +static int __init cpufreq_sched_init(void) +{ + int cpu; + + for_each_cpu(cpu, cpu_possible_mask) { + per_cpu(governor_started, cpu) = 0; + } + mutex_init(&gov_list_lock); + freq_change_task = kthread_run(cpufreq_sched_thread, NULL, "kcpufreq_sched_task"); + if (IS_ERR_OR_NULL(freq_change_task)) { + pr_err("%s: failed to create kcpufreq_sched_task thread\n", __func__); + return -ENOMEM; + } + init_irq_work(&irq_work, cpufreq_sched_irq_work); + return cpufreq_register_governor(&cpufreq_gov_sched); +} + +static void __exit cpufreq_sched_exit(void) +{ + kthread_stop(freq_change_task); + cpufreq_unregister_governor(&cpufreq_gov_sched); +} + +/* Try to make this the default governor */ +fs_initcall(cpufreq_sched_init); + +MODULE_LICENSE("GPL v2"); diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c index 61d9e72c9df96..981fcd7dc394e 100644 --- a/kernel/sched/cpupri.c +++ b/kernel/sched/cpupri.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "cpupri.h" /* Convert between a 140 based task->prio, and our 102 based cpupri */ @@ -62,7 +63,7 @@ static int convert_prio(int prio) * any discrepancies created by racing against the uncertainty of the current * priority configuration. * - * Returns: (int)bool - CPUs were found + * Return: (int)bool - CPUs were found */ int cpupri_find(struct cpupri *cp, struct task_struct *p, struct cpumask *lowest_mask) @@ -202,7 +203,7 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri) * cpupri_init - initialize the cpupri structure * @cp: The cpupri context * - * Returns: -ENOMEM if memory fails. + * Return: -ENOMEM on memory allocation failure. */ int cpupri_init(struct cpupri *cp) { @@ -218,8 +219,13 @@ int cpupri_init(struct cpupri *cp) goto cleanup; } + cp->cpu_to_pri = kcalloc(nr_cpu_ids, sizeof(int), GFP_KERNEL); + if (!cp->cpu_to_pri) + goto cleanup; + for_each_possible_cpu(i) cp->cpu_to_pri[i] = CPUPRI_INVALID; + return 0; cleanup: @@ -236,6 +242,7 @@ void cpupri_cleanup(struct cpupri *cp) { int i; + kfree(cp->cpu_to_pri); for (i = 0; i < CPUPRI_NR_PRIORITIES; i++) free_cpumask_var(cp->pri_to_cpu[i].mask); } diff --git a/kernel/sched/cpupri.h b/kernel/sched/cpupri.h index f6d756173491c..6b033347fdfd1 100644 --- a/kernel/sched/cpupri.h +++ b/kernel/sched/cpupri.h @@ -17,7 +17,7 @@ struct cpupri_vec { struct cpupri { struct cpupri_vec pri_to_cpu[CPUPRI_NR_PRIORITIES]; - int cpu_to_pri[NR_CPUS]; + int *cpu_to_pri; }; #ifdef CONFIG_SMP diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 199dda8db9a0e..481ded956affa 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -50,7 +50,6 @@ void irqtime_account_irq(struct task_struct *curr) unsigned long flags; s64 delta; int cpu; - u64 wallclock; if (!sched_clock_irqtime) return; @@ -58,8 +57,7 @@ void irqtime_account_irq(struct task_struct *curr) local_irq_save(flags); cpu = smp_processor_id(); - wallclock = sched_clock_cpu(cpu); - delta = wallclock - __this_cpu_read(irq_start_time); + delta = sched_clock_cpu(cpu) - __this_cpu_read(irq_start_time); __this_cpu_add(irq_start_time, delta); irq_time_write_begin(); @@ -69,14 +67,10 @@ void irqtime_account_irq(struct task_struct *curr) * in that case, so as not to confuse scheduler with a special task * that do not consume any time, but still wants to run. */ - if (hardirq_count()) { + if (hardirq_count()) __this_cpu_add(cpu_hardirq_time, delta); - sched_account_irqtime(cpu, curr, delta, wallclock); - } else if (in_serving_softirq() && curr != this_cpu_ksoftirqd()) { + else if (in_serving_softirq() && curr != this_cpu_ksoftirqd()) __this_cpu_add(cpu_softirq_time, delta); - sched_account_irqtime(cpu, curr, delta, wallclock); - } - irq_time_write_end(); local_irq_restore(flags); @@ -128,7 +122,7 @@ static inline void task_group_account_field(struct task_struct *p, int index, * is the only cgroup, then nothing else should be necessary. * */ - __get_cpu_var(kernel_cpustat).cpustat[index] += tmp; + __this_cpu_add(kernel_cpustat.cpustat[index], tmp); cpuacct_account_field(p, index, tmp); } @@ -149,7 +143,7 @@ void account_user_time(struct task_struct *p, cputime_t cputime, p->utimescaled += cputime_scaled; account_group_user_time(p, cputime); - index = (TASK_NICE(p) > 0) ? CPUTIME_NICE : CPUTIME_USER; + index = (task_nice(p) > 0) ? CPUTIME_NICE : CPUTIME_USER; /* Add user time to cpustat. */ task_group_account_field(p, index, (__force u64) cputime); @@ -179,7 +173,7 @@ static void account_guest_time(struct task_struct *p, cputime_t cputime, p->gtime += cputime; /* Add guest time to cpustat. */ - if (TASK_NICE(p) > 0) { + if (task_nice(p) > 0) { cpustat[CPUTIME_NICE] += (__force u64) cputime; cpustat[CPUTIME_GUEST_NICE] += (__force u64) cputime; } else { @@ -271,16 +265,22 @@ static __always_inline bool steal_account_process_tick(void) { #ifdef CONFIG_PARAVIRT if (static_key_false(¶virt_steal_enabled)) { - u64 steal, st = 0; + u64 steal; + cputime_t steal_ct; steal = paravirt_steal_clock(smp_processor_id()); steal -= this_rq()->prev_steal_time; - st = steal_ticks(steal); - this_rq()->prev_steal_time += st * TICK_NSEC; + /* + * cputime_t may be less precise than nsecs (eg: if it's + * based on jiffies). Lets cast the result to cputime + * granularity and account the rest on the next rounds. + */ + steal_ct = nsecs_to_cputime(steal); + this_rq()->prev_steal_time += cputime_to_nsecs(steal_ct); - account_steal_time(st); - return st; + account_steal_time(steal_ct); + return steal_ct; } #endif return false; @@ -391,11 +391,8 @@ static inline void irqtime_account_process_tick(struct task_struct *p, int user_ #ifdef CONFIG_VIRT_CPU_ACCOUNTING #ifndef __ARCH_HAS_VTIME_TASK_SWITCH -void vtime_task_switch(struct task_struct *prev) +void vtime_common_task_switch(struct task_struct *prev) { - if (!vtime_accounting_enabled()) - return; - if (is_idle_task(prev)) vtime_account_idle(prev); else @@ -417,11 +414,8 @@ void vtime_task_switch(struct task_struct *prev) * vtime_account(). */ #ifndef __ARCH_HAS_VTIME_ACCOUNT -void vtime_account_irq_enter(struct task_struct *tsk) +void vtime_common_account_irq_enter(struct task_struct *tsk) { - if (!vtime_accounting_enabled()) - return; - if (!in_interrupt()) { /* * If we interrupted user, context_tracking_in_user() @@ -441,7 +435,7 @@ void vtime_account_irq_enter(struct task_struct *tsk) } vtime_account_system(tsk); } -EXPORT_SYMBOL_GPL(vtime_account_irq_enter); +EXPORT_SYMBOL_GPL(vtime_common_account_irq_enter); #endif /* __ARCH_HAS_VTIME_ACCOUNT */ #endif /* CONFIG_VIRT_CPU_ACCOUNTING */ @@ -528,9 +522,8 @@ static cputime_t scale_stime(u64 stime, u64 rtime, u64 total) for (;;) { /* Make sure "rtime" is the bigger of stime/rtime */ - if (stime > rtime) { - u64 tmp = rtime; rtime = stime; stime = tmp; - } + if (stime > rtime) + swap(rtime, stime); /* Make sure 'total' fits in 32 bits */ if (total >> 32) @@ -637,9 +630,6 @@ void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st) cputime_adjust(&cputime, &p->prev_cputime, ut, st); } -/* - * Must be called with siglock held. - */ void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st) { struct task_cputime cputime; @@ -681,23 +671,17 @@ static void __vtime_account_system(struct task_struct *tsk) void vtime_account_system(struct task_struct *tsk) { - if (!vtime_accounting_enabled()) - return; - write_seqlock(&tsk->vtime_seqlock); __vtime_account_system(tsk); write_sequnlock(&tsk->vtime_seqlock); } -void vtime_account_irq_exit(struct task_struct *tsk) +void vtime_gen_account_irq_exit(struct task_struct *tsk) { - if (!vtime_accounting_enabled()) - return; - write_seqlock(&tsk->vtime_seqlock); + __vtime_account_system(tsk); if (context_tracking_in_user()) tsk->vtime_snap_whence = VTIME_USER; - __vtime_account_system(tsk); write_sequnlock(&tsk->vtime_seqlock); } @@ -705,12 +689,8 @@ void vtime_account_user(struct task_struct *tsk) { cputime_t delta_cpu; - if (!vtime_accounting_enabled()) - return; - - delta_cpu = get_vtime_delta(tsk); - write_seqlock(&tsk->vtime_seqlock); + delta_cpu = get_vtime_delta(tsk); tsk->vtime_snap_whence = VTIME_SYS; account_user_time(tsk, delta_cpu, cputime_to_scaled(delta_cpu)); write_sequnlock(&tsk->vtime_seqlock); @@ -718,22 +698,27 @@ void vtime_account_user(struct task_struct *tsk) void vtime_user_enter(struct task_struct *tsk) { - if (!vtime_accounting_enabled()) - return; - write_seqlock(&tsk->vtime_seqlock); - tsk->vtime_snap_whence = VTIME_USER; __vtime_account_system(tsk); + tsk->vtime_snap_whence = VTIME_USER; write_sequnlock(&tsk->vtime_seqlock); } void vtime_guest_enter(struct task_struct *tsk) { + /* + * The flags must be updated under the lock with + * the vtime_snap flush and update. + * That enforces a right ordering and update sequence + * synchronization against the reader (task_gtime()) + * that can thus safely catch up with a tickless delta. + */ write_seqlock(&tsk->vtime_seqlock); __vtime_account_system(tsk); current->flags |= PF_VCPU; write_sequnlock(&tsk->vtime_seqlock); } +EXPORT_SYMBOL_GPL(vtime_guest_enter); void vtime_guest_exit(struct task_struct *tsk) { @@ -742,6 +727,7 @@ void vtime_guest_exit(struct task_struct *tsk) current->flags &= ~PF_VCPU; write_sequnlock(&tsk->vtime_seqlock); } +EXPORT_SYMBOL_GPL(vtime_guest_exit); void vtime_account_idle(struct task_struct *tsk) { @@ -750,11 +736,6 @@ void vtime_account_idle(struct task_struct *tsk) account_idle_time(delta_cpu); } -bool vtime_accounting_enabled(void) -{ - return context_tracking_active(); -} - void arch_vtime_task_switch(struct task_struct *prev) { write_seqlock(&prev->vtime_seqlock); diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c new file mode 100644 index 0000000000000..fc0fd03698610 --- /dev/null +++ b/kernel/sched/deadline.c @@ -0,0 +1,1689 @@ +/* + * Deadline Scheduling Class (SCHED_DEADLINE) + * + * Earliest Deadline First (EDF) + Constant Bandwidth Server (CBS). + * + * Tasks that periodically executes their instances for less than their + * runtime won't miss any of their deadlines. + * Tasks that are not periodic or sporadic or that tries to execute more + * than their reserved bandwidth will be slowed down (and may potentially + * miss some of their deadlines), and won't affect any other task. + * + * Copyright (C) 2012 Dario Faggioli , + * Juri Lelli , + * Michael Trimarchi , + * Fabio Checconi + */ +#include "sched.h" + +#include + +struct dl_bandwidth def_dl_bandwidth; + +static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se) +{ + return container_of(dl_se, struct task_struct, dl); +} + +static inline struct rq *rq_of_dl_rq(struct dl_rq *dl_rq) +{ + return container_of(dl_rq, struct rq, dl); +} + +static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se) +{ + struct task_struct *p = dl_task_of(dl_se); + struct rq *rq = task_rq(p); + + return &rq->dl; +} + +static inline int on_dl_rq(struct sched_dl_entity *dl_se) +{ + return !RB_EMPTY_NODE(&dl_se->rb_node); +} + +static inline int is_leftmost(struct task_struct *p, struct dl_rq *dl_rq) +{ + struct sched_dl_entity *dl_se = &p->dl; + + return dl_rq->rb_leftmost == &dl_se->rb_node; +} + +void init_dl_bandwidth(struct dl_bandwidth *dl_b, u64 period, u64 runtime) +{ + raw_spin_lock_init(&dl_b->dl_runtime_lock); + dl_b->dl_period = period; + dl_b->dl_runtime = runtime; +} + +void init_dl_bw(struct dl_bw *dl_b) +{ + raw_spin_lock_init(&dl_b->lock); + raw_spin_lock(&def_dl_bandwidth.dl_runtime_lock); + if (global_rt_runtime() == RUNTIME_INF) + dl_b->bw = -1; + else + dl_b->bw = to_ratio(global_rt_period(), global_rt_runtime()); + raw_spin_unlock(&def_dl_bandwidth.dl_runtime_lock); + dl_b->total_bw = 0; +} + +void init_dl_rq(struct dl_rq *dl_rq, struct rq *rq) +{ + dl_rq->rb_root = RB_ROOT; + +#ifdef CONFIG_SMP + /* zero means no -deadline tasks */ + dl_rq->earliest_dl.curr = dl_rq->earliest_dl.next = 0; + + dl_rq->dl_nr_migratory = 0; + dl_rq->overloaded = 0; + dl_rq->pushable_dl_tasks_root = RB_ROOT; +#else + init_dl_bw(&dl_rq->dl_bw); +#endif +} + +#ifdef CONFIG_SMP + +static inline int dl_overloaded(struct rq *rq) +{ + return atomic_read(&rq->rd->dlo_count); +} + +static inline void dl_set_overload(struct rq *rq) +{ + if (!rq->online) + return; + + cpumask_set_cpu(rq->cpu, rq->rd->dlo_mask); + /* + * Must be visible before the overload count is + * set (as in sched_rt.c). + * + * Matched by the barrier in pull_dl_task(). + */ + smp_wmb(); + atomic_inc(&rq->rd->dlo_count); +} + +static inline void dl_clear_overload(struct rq *rq) +{ + if (!rq->online) + return; + + atomic_dec(&rq->rd->dlo_count); + cpumask_clear_cpu(rq->cpu, rq->rd->dlo_mask); +} + +static void update_dl_migration(struct dl_rq *dl_rq) +{ + if (dl_rq->dl_nr_migratory && dl_rq->dl_nr_running > 1) { + if (!dl_rq->overloaded) { + dl_set_overload(rq_of_dl_rq(dl_rq)); + dl_rq->overloaded = 1; + } + } else if (dl_rq->overloaded) { + dl_clear_overload(rq_of_dl_rq(dl_rq)); + dl_rq->overloaded = 0; + } +} + +static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) +{ + struct task_struct *p = dl_task_of(dl_se); + + if (p->nr_cpus_allowed > 1) + dl_rq->dl_nr_migratory++; + + update_dl_migration(dl_rq); +} + +static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) +{ + struct task_struct *p = dl_task_of(dl_se); + + if (p->nr_cpus_allowed > 1) + dl_rq->dl_nr_migratory--; + + update_dl_migration(dl_rq); +} + +/* + * The list of pushable -deadline task is not a plist, like in + * sched_rt.c, it is an rb-tree with tasks ordered by deadline. + */ +static void enqueue_pushable_dl_task(struct rq *rq, struct task_struct *p) +{ + struct dl_rq *dl_rq = &rq->dl; + struct rb_node **link = &dl_rq->pushable_dl_tasks_root.rb_node; + struct rb_node *parent = NULL; + struct task_struct *entry; + int leftmost = 1; + + BUG_ON(!RB_EMPTY_NODE(&p->pushable_dl_tasks)); + + while (*link) { + parent = *link; + entry = rb_entry(parent, struct task_struct, + pushable_dl_tasks); + if (dl_entity_preempt(&p->dl, &entry->dl)) + link = &parent->rb_left; + else { + link = &parent->rb_right; + leftmost = 0; + } + } + + if (leftmost) + dl_rq->pushable_dl_tasks_leftmost = &p->pushable_dl_tasks; + + rb_link_node(&p->pushable_dl_tasks, parent, link); + rb_insert_color(&p->pushable_dl_tasks, &dl_rq->pushable_dl_tasks_root); +} + +static void dequeue_pushable_dl_task(struct rq *rq, struct task_struct *p) +{ + struct dl_rq *dl_rq = &rq->dl; + + if (RB_EMPTY_NODE(&p->pushable_dl_tasks)) + return; + + if (dl_rq->pushable_dl_tasks_leftmost == &p->pushable_dl_tasks) { + struct rb_node *next_node; + + next_node = rb_next(&p->pushable_dl_tasks); + dl_rq->pushable_dl_tasks_leftmost = next_node; + } + + rb_erase(&p->pushable_dl_tasks, &dl_rq->pushable_dl_tasks_root); + RB_CLEAR_NODE(&p->pushable_dl_tasks); +} + +static inline int has_pushable_dl_tasks(struct rq *rq) +{ + return !RB_EMPTY_ROOT(&rq->dl.pushable_dl_tasks_root); +} + +static int push_dl_task(struct rq *rq); + +static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev) +{ + return dl_task(prev); +} + +static inline void set_post_schedule(struct rq *rq) +{ + rq->post_schedule = has_pushable_dl_tasks(rq); +} + +#else + +static inline +void enqueue_pushable_dl_task(struct rq *rq, struct task_struct *p) +{ +} + +static inline +void dequeue_pushable_dl_task(struct rq *rq, struct task_struct *p) +{ +} + +static inline +void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) +{ +} + +static inline +void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) +{ +} + +static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev) +{ + return false; +} + +static inline int pull_dl_task(struct rq *rq) +{ + return 0; +} + +static inline void set_post_schedule(struct rq *rq) +{ +} +#endif /* CONFIG_SMP */ + +static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags); +static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags); +static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p, + int flags); + +/* + * We are being explicitly informed that a new instance is starting, + * and this means that: + * - the absolute deadline of the entity has to be placed at + * current time + relative deadline; + * - the runtime of the entity has to be set to the maximum value. + * + * The capability of specifying such event is useful whenever a -deadline + * entity wants to (try to!) synchronize its behaviour with the scheduler's + * one, and to (try to!) reconcile itself with its own scheduling + * parameters. + */ +static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se, + struct sched_dl_entity *pi_se) +{ + struct dl_rq *dl_rq = dl_rq_of_se(dl_se); + struct rq *rq = rq_of_dl_rq(dl_rq); + + WARN_ON(!dl_se->dl_new || dl_se->dl_throttled); + + /* + * We use the regular wall clock time to set deadlines in the + * future; in fact, we must consider execution overheads (time + * spent on hardirq context, etc.). + */ + dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; + dl_se->runtime = pi_se->dl_runtime; + dl_se->dl_new = 0; +} + +/* + * Pure Earliest Deadline First (EDF) scheduling does not deal with the + * possibility of a entity lasting more than what it declared, and thus + * exhausting its runtime. + * + * Here we are interested in making runtime overrun possible, but we do + * not want a entity which is misbehaving to affect the scheduling of all + * other entities. + * Therefore, a budgeting strategy called Constant Bandwidth Server (CBS) + * is used, in order to confine each entity within its own bandwidth. + * + * This function deals exactly with that, and ensures that when the runtime + * of a entity is replenished, its deadline is also postponed. That ensures + * the overrunning entity can't interfere with other entity in the system and + * can't make them miss their deadlines. Reasons why this kind of overruns + * could happen are, typically, a entity voluntarily trying to overcome its + * runtime, or it just underestimated it during sched_setattr(). + */ +static void replenish_dl_entity(struct sched_dl_entity *dl_se, + struct sched_dl_entity *pi_se) +{ + struct dl_rq *dl_rq = dl_rq_of_se(dl_se); + struct rq *rq = rq_of_dl_rq(dl_rq); + + BUG_ON(pi_se->dl_runtime <= 0); + + /* + * This could be the case for a !-dl task that is boosted. + * Just go with full inherited parameters. + */ + if (dl_se->dl_deadline == 0) { + dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; + dl_se->runtime = pi_se->dl_runtime; + } + + /* + * We keep moving the deadline away until we get some + * available runtime for the entity. This ensures correct + * handling of situations where the runtime overrun is + * arbitrary large. + */ + while (dl_se->runtime <= 0) { + dl_se->deadline += pi_se->dl_period; + dl_se->runtime += pi_se->dl_runtime; + } + + /* + * At this point, the deadline really should be "in + * the future" with respect to rq->clock. If it's + * not, we are, for some reason, lagging too much! + * Anyway, after having warn userspace abut that, + * we still try to keep the things running by + * resetting the deadline and the budget of the + * entity. + */ + if (dl_time_before(dl_se->deadline, rq_clock(rq))) { + //TJK printk_deferred_once("sched: DL replenish lagged to much\n"); + dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; + dl_se->runtime = pi_se->dl_runtime; + } +} + +/* + * Here we check if --at time t-- an entity (which is probably being + * [re]activated or, in general, enqueued) can use its remaining runtime + * and its current deadline _without_ exceeding the bandwidth it is + * assigned (function returns true if it can't). We are in fact applying + * one of the CBS rules: when a task wakes up, if the residual runtime + * over residual deadline fits within the allocated bandwidth, then we + * can keep the current (absolute) deadline and residual budget without + * disrupting the schedulability of the system. Otherwise, we should + * refill the runtime and set the deadline a period in the future, + * because keeping the current (absolute) deadline of the task would + * result in breaking guarantees promised to other tasks (refer to + * Documentation/scheduler/sched-deadline.txt for more informations). + * + * This function returns true if: + * + * runtime / (deadline - t) > dl_runtime / dl_period , + * + * IOW we can't recycle current parameters. + * + * Notice that the bandwidth check is done against the period. For + * task with deadline equal to period this is the same of using + * dl_deadline instead of dl_period in the equation above. + */ +static bool dl_entity_overflow(struct sched_dl_entity *dl_se, + struct sched_dl_entity *pi_se, u64 t) +{ + u64 left, right; + + /* + * left and right are the two sides of the equation above, + * after a bit of shuffling to use multiplications instead + * of divisions. + * + * Note that none of the time values involved in the two + * multiplications are absolute: dl_deadline and dl_runtime + * are the relative deadline and the maximum runtime of each + * instance, runtime is the runtime left for the last instance + * and (deadline - t), since t is rq->clock, is the time left + * to the (absolute) deadline. Even if overflowing the u64 type + * is very unlikely to occur in both cases, here we scale down + * as we want to avoid that risk at all. Scaling down by 10 + * means that we reduce granularity to 1us. We are fine with it, + * since this is only a true/false check and, anyway, thinking + * of anything below microseconds resolution is actually fiction + * (but still we want to give the user that illusion >;). + */ + left = (pi_se->dl_period >> DL_SCALE) * (dl_se->runtime >> DL_SCALE); + right = ((dl_se->deadline - t) >> DL_SCALE) * + (pi_se->dl_runtime >> DL_SCALE); + + return dl_time_before(right, left); +} + +/* + * When a -deadline entity is queued back on the runqueue, its runtime and + * deadline might need updating. + * + * The policy here is that we update the deadline of the entity only if: + * - the current deadline is in the past, + * - using the remaining runtime with the current deadline would make + * the entity exceed its bandwidth. + */ +static void update_dl_entity(struct sched_dl_entity *dl_se, + struct sched_dl_entity *pi_se) +{ + struct dl_rq *dl_rq = dl_rq_of_se(dl_se); + struct rq *rq = rq_of_dl_rq(dl_rq); + + /* + * The arrival of a new instance needs special treatment, i.e., + * the actual scheduling parameters have to be "renewed". + */ + if (dl_se->dl_new) { + setup_new_dl_entity(dl_se, pi_se); + return; + } + + if (dl_time_before(dl_se->deadline, rq_clock(rq)) || + dl_entity_overflow(dl_se, pi_se, rq_clock(rq))) { + dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; + dl_se->runtime = pi_se->dl_runtime; + } +} + +/* + * If the entity depleted all its runtime, and if we want it to sleep + * while waiting for some new execution time to become available, we + * set the bandwidth enforcement timer to the replenishment instant + * and try to activate it. + * + * Notice that it is important for the caller to know if the timer + * actually started or not (i.e., the replenishment instant is in + * the future or in the past). + */ +static int start_dl_timer(struct sched_dl_entity *dl_se, bool boosted) +{ + struct dl_rq *dl_rq = dl_rq_of_se(dl_se); + struct rq *rq = rq_of_dl_rq(dl_rq); + ktime_t now, act; + ktime_t soft, hard; + unsigned long range; + s64 delta; + + if (boosted) + return 0; + /* + * We want the timer to fire at the deadline, but considering + * that it is actually coming from rq->clock and not from + * hrtimer's time base reading. + */ + act = ns_to_ktime(dl_se->deadline); + now = hrtimer_cb_get_time(&dl_se->dl_timer); + delta = ktime_to_ns(now) - rq_clock(rq); + act = ktime_add_ns(act, delta); + + /* + * If the expiry time already passed, e.g., because the value + * chosen as the deadline is too small, don't even try to + * start the timer in the past! + */ + if (ktime_us_delta(act, now) < 0) + return 0; + + hrtimer_set_expires(&dl_se->dl_timer, act); + + soft = hrtimer_get_softexpires(&dl_se->dl_timer); + hard = hrtimer_get_expires(&dl_se->dl_timer); + range = ktime_to_ns(ktime_sub(hard, soft)); + __hrtimer_start_range_ns(&dl_se->dl_timer, soft, + range, HRTIMER_MODE_ABS, 0); + + return hrtimer_active(&dl_se->dl_timer); +} + +/* + * This is the bandwidth enforcement timer callback. If here, we know + * a task is not on its dl_rq, since the fact that the timer was running + * means the task is throttled and needs a runtime replenishment. + * + * However, what we actually do depends on the fact the task is active, + * (it is on its rq) or has been removed from there by a call to + * dequeue_task_dl(). In the former case we must issue the runtime + * replenishment and add the task back to the dl_rq; in the latter, we just + * do nothing but clearing dl_throttled, so that runtime and deadline + * updating (and the queueing back to dl_rq) will be done by the + * next call to enqueue_task_dl(). + */ +static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) +{ + struct sched_dl_entity *dl_se = container_of(timer, + struct sched_dl_entity, + dl_timer); + struct task_struct *p = dl_task_of(dl_se); + struct rq *rq; +again: + rq = task_rq(p); + raw_spin_lock(&rq->lock); + + if (rq != task_rq(p)) { + /* Task was moved, retrying. */ + raw_spin_unlock(&rq->lock); + goto again; + } + + /* + * We need to take care of several possible races here: + * + * - the task might have changed its scheduling policy + * to something different than SCHED_DEADLINE + * - the task might have changed its reservation parameters + * (through sched_setattr()) + * - the task might have been boosted by someone else and + * might be in the boosting/deboosting path + * + * In all this cases we bail out, as the task is already + * in the runqueue or is going to be enqueued back anyway. + */ + if (!dl_task(p) || dl_se->dl_new || + dl_se->dl_boosted || !dl_se->dl_throttled) + goto unlock; + + sched_clock_tick(); + update_rq_clock(rq); + dl_se->dl_throttled = 0; + dl_se->dl_yielded = 0; + if (task_on_rq_queued(p)) { + enqueue_task_dl(rq, p, ENQUEUE_REPLENISH); + if (dl_task(rq->curr)) + check_preempt_curr_dl(rq, p, 0); + else + resched_curr(rq); +#ifdef CONFIG_SMP + /* + * Queueing this task back might have overloaded rq, + * check if we need to kick someone away. + */ + if (has_pushable_dl_tasks(rq)) + push_dl_task(rq); +#endif + } +unlock: + raw_spin_unlock(&rq->lock); + + return HRTIMER_NORESTART; +} + +void init_dl_task_timer(struct sched_dl_entity *dl_se) +{ + struct hrtimer *timer = &dl_se->dl_timer; + + if (hrtimer_active(timer)) { + hrtimer_try_to_cancel(timer); + return; + } + + hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + timer->function = dl_task_timer; +} + +static +int dl_runtime_exceeded(struct rq *rq, struct sched_dl_entity *dl_se) +{ + return (dl_se->runtime <= 0); +} + +extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq); + +/* + * Update the current task's runtime statistics (provided it is still + * a -deadline task and has not been removed from the dl_rq). + */ +static void update_curr_dl(struct rq *rq) +{ + struct task_struct *curr = rq->curr; + struct sched_dl_entity *dl_se = &curr->dl; + u64 delta_exec; + + if (!dl_task(curr) || !on_dl_rq(dl_se)) + return; + + /* + * Consumed budget is computed considering the time as + * observed by schedulable tasks (excluding time spent + * in hardirq context, etc.). Deadlines are instead + * computed using hard walltime. This seems to be the more + * natural solution, but the full ramifications of this + * approach need further study. + */ + delta_exec = rq_clock_task(rq) - curr->se.exec_start; + if (unlikely((s64)delta_exec <= 0)) + return; + + schedstat_set(curr->se.statistics.exec_max, + max(curr->se.statistics.exec_max, delta_exec)); + + curr->se.sum_exec_runtime += delta_exec; + account_group_exec_runtime(curr, delta_exec); + + curr->se.exec_start = rq_clock_task(rq); + cpuacct_charge(curr, delta_exec); + + sched_rt_avg_update(rq, delta_exec); + + dl_se->runtime -= delta_exec; + if (dl_runtime_exceeded(rq, dl_se)) { + __dequeue_task_dl(rq, curr, 0); + if (likely(start_dl_timer(dl_se, curr->dl.dl_boosted))) + dl_se->dl_throttled = 1; + else + enqueue_task_dl(rq, curr, ENQUEUE_REPLENISH); + + if (!is_leftmost(curr, &rq->dl)) + resched_curr(rq); + } + + /* + * Because -- for now -- we share the rt bandwidth, we need to + * account our runtime there too, otherwise actual rt tasks + * would be able to exceed the shared quota. + * + * Account to the root rt group for now. + * + * The solution we're working towards is having the RT groups scheduled + * using deadline servers -- however there's a few nasties to figure + * out before that can happen. + */ + if (rt_bandwidth_enabled()) { + struct rt_rq *rt_rq = &rq->rt; + + raw_spin_lock(&rt_rq->rt_runtime_lock); + /* + * We'll let actual RT tasks worry about the overflow here, we + * have our own CBS to keep us inline; only account when RT + * bandwidth is relevant. + */ + if (sched_rt_bandwidth_account(rt_rq)) + rt_rq->rt_time += delta_exec; + raw_spin_unlock(&rt_rq->rt_runtime_lock); + } +} + +#ifdef CONFIG_SMP + +static struct task_struct *pick_next_earliest_dl_task(struct rq *rq, int cpu); + +static inline u64 next_deadline(struct rq *rq) +{ + struct task_struct *next = pick_next_earliest_dl_task(rq, rq->cpu); + + if (next && dl_prio(next->prio)) + return next->dl.deadline; + else + return 0; +} + +static void inc_dl_deadline(struct dl_rq *dl_rq, u64 deadline) +{ + struct rq *rq = rq_of_dl_rq(dl_rq); + + if (dl_rq->earliest_dl.curr == 0 || + dl_time_before(deadline, dl_rq->earliest_dl.curr)) { + /* + * If the dl_rq had no -deadline tasks, or if the new task + * has shorter deadline than the current one on dl_rq, we + * know that the previous earliest becomes our next earliest, + * as the new task becomes the earliest itself. + */ + dl_rq->earliest_dl.next = dl_rq->earliest_dl.curr; + dl_rq->earliest_dl.curr = deadline; + cpudl_set(&rq->rd->cpudl, rq->cpu, deadline, 1); + } else if (dl_rq->earliest_dl.next == 0 || + dl_time_before(deadline, dl_rq->earliest_dl.next)) { + /* + * On the other hand, if the new -deadline task has a + * a later deadline than the earliest one on dl_rq, but + * it is earlier than the next (if any), we must + * recompute the next-earliest. + */ + dl_rq->earliest_dl.next = next_deadline(rq); + } +} + +static void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline) +{ + struct rq *rq = rq_of_dl_rq(dl_rq); + + /* + * Since we may have removed our earliest (and/or next earliest) + * task we must recompute them. + */ + if (!dl_rq->dl_nr_running) { + dl_rq->earliest_dl.curr = 0; + dl_rq->earliest_dl.next = 0; + cpudl_set(&rq->rd->cpudl, rq->cpu, 0, 0); + } else { + struct rb_node *leftmost = dl_rq->rb_leftmost; + struct sched_dl_entity *entry; + + entry = rb_entry(leftmost, struct sched_dl_entity, rb_node); + dl_rq->earliest_dl.curr = entry->deadline; + dl_rq->earliest_dl.next = next_deadline(rq); + cpudl_set(&rq->rd->cpudl, rq->cpu, entry->deadline, 1); + } +} + +#else + +static inline void inc_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {} +static inline void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {} + +#endif /* CONFIG_SMP */ + +static inline +void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) +{ + int prio = dl_task_of(dl_se)->prio; + u64 deadline = dl_se->deadline; + + WARN_ON(!dl_prio(prio)); + dl_rq->dl_nr_running++; + add_nr_running(rq_of_dl_rq(dl_rq), 1); + + inc_dl_deadline(dl_rq, deadline); + inc_dl_migration(dl_se, dl_rq); +} + +static inline +void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) +{ + int prio = dl_task_of(dl_se)->prio; + + WARN_ON(!dl_prio(prio)); + WARN_ON(!dl_rq->dl_nr_running); + dl_rq->dl_nr_running--; + sub_nr_running(rq_of_dl_rq(dl_rq), 1); + + dec_dl_deadline(dl_rq, dl_se->deadline); + dec_dl_migration(dl_se, dl_rq); +} + +static void __enqueue_dl_entity(struct sched_dl_entity *dl_se) +{ + struct dl_rq *dl_rq = dl_rq_of_se(dl_se); + struct rb_node **link = &dl_rq->rb_root.rb_node; + struct rb_node *parent = NULL; + struct sched_dl_entity *entry; + int leftmost = 1; + + BUG_ON(!RB_EMPTY_NODE(&dl_se->rb_node)); + + while (*link) { + parent = *link; + entry = rb_entry(parent, struct sched_dl_entity, rb_node); + if (dl_time_before(dl_se->deadline, entry->deadline)) + link = &parent->rb_left; + else { + link = &parent->rb_right; + leftmost = 0; + } + } + + if (leftmost) + dl_rq->rb_leftmost = &dl_se->rb_node; + + rb_link_node(&dl_se->rb_node, parent, link); + rb_insert_color(&dl_se->rb_node, &dl_rq->rb_root); + + inc_dl_tasks(dl_se, dl_rq); +} + +static void __dequeue_dl_entity(struct sched_dl_entity *dl_se) +{ + struct dl_rq *dl_rq = dl_rq_of_se(dl_se); + + if (RB_EMPTY_NODE(&dl_se->rb_node)) + return; + + if (dl_rq->rb_leftmost == &dl_se->rb_node) { + struct rb_node *next_node; + + next_node = rb_next(&dl_se->rb_node); + dl_rq->rb_leftmost = next_node; + } + + rb_erase(&dl_se->rb_node, &dl_rq->rb_root); + RB_CLEAR_NODE(&dl_se->rb_node); + + dec_dl_tasks(dl_se, dl_rq); +} + +static void +enqueue_dl_entity(struct sched_dl_entity *dl_se, + struct sched_dl_entity *pi_se, int flags) +{ + BUG_ON(on_dl_rq(dl_se)); + + /* + * If this is a wakeup or a new instance, the scheduling + * parameters of the task might need updating. Otherwise, + * we want a replenishment of its runtime. + */ + if (dl_se->dl_new || flags & ENQUEUE_WAKEUP) + update_dl_entity(dl_se, pi_se); + else if (flags & ENQUEUE_REPLENISH) + replenish_dl_entity(dl_se, pi_se); + + __enqueue_dl_entity(dl_se); +} + +static void dequeue_dl_entity(struct sched_dl_entity *dl_se) +{ + __dequeue_dl_entity(dl_se); +} + +static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) +{ + struct task_struct *pi_task = rt_mutex_get_top_task(p); + struct sched_dl_entity *pi_se = &p->dl; + + /* + * Use the scheduling parameters of the top pi-waiter + * task if we have one and its (relative) deadline is + * smaller than our one... OTW we keep our runtime and + * deadline. + */ + if (pi_task && p->dl.dl_boosted && dl_prio(pi_task->normal_prio)) { + pi_se = &pi_task->dl; + } else if (!dl_prio(p->normal_prio)) { + /* + * Special case in which we have a !SCHED_DEADLINE task + * that is going to be deboosted, but exceedes its + * runtime while doing so. No point in replenishing + * it, as it's going to return back to its original + * scheduling class after this. + */ + BUG_ON(!p->dl.dl_boosted || flags != ENQUEUE_REPLENISH); + return; + } + + /* + * If p is throttled, we do nothing. In fact, if it exhausted + * its budget it needs a replenishment and, since it now is on + * its rq, the bandwidth timer callback (which clearly has not + * run yet) will take care of this. + */ + if (p->dl.dl_throttled) + return; + + enqueue_dl_entity(&p->dl, pi_se, flags); + + if (!task_current(rq, p) && p->nr_cpus_allowed > 1) + enqueue_pushable_dl_task(rq, p); +} + +static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags) +{ + dequeue_dl_entity(&p->dl); + dequeue_pushable_dl_task(rq, p); +} + +static void dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags) +{ + update_curr_dl(rq); + __dequeue_task_dl(rq, p, flags); +} + +/* + * Yield task semantic for -deadline tasks is: + * + * get off from the CPU until our next instance, with + * a new runtime. This is of little use now, since we + * don't have a bandwidth reclaiming mechanism. Anyway, + * bandwidth reclaiming is planned for the future, and + * yield_task_dl will indicate that some spare budget + * is available for other task instances to use it. + */ +static void yield_task_dl(struct rq *rq) +{ + struct task_struct *p = rq->curr; + + /* + * We make the task go to sleep until its current deadline by + * forcing its runtime to zero. This way, update_curr_dl() stops + * it and the bandwidth timer will wake it up and will give it + * new scheduling parameters (thanks to dl_yielded=1). + */ + if (p->dl.runtime > 0) { + rq->curr->dl.dl_yielded = 1; + p->dl.runtime = 0; + } + update_curr_dl(rq); +} + +#ifdef CONFIG_SMP + +static int find_later_rq(struct task_struct *task); + +static int +select_task_rq_dl(struct task_struct *p, int cpu, int sd_flag, int flags) +{ + struct task_struct *curr; + struct rq *rq; + + if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK) + goto out; + + rq = cpu_rq(cpu); + + rcu_read_lock(); + curr = ACCESS_ONCE(rq->curr); /* unlocked access */ + + /* + * If we are dealing with a -deadline task, we must + * decide where to wake it up. + * If it has a later deadline and the current task + * on this rq can't move (provided the waking task + * can!) we prefer to send it somewhere else. On the + * other hand, if it has a shorter deadline, we + * try to make it stay here, it might be important. + */ + if (unlikely(dl_task(curr)) && + (curr->nr_cpus_allowed < 2 || + !dl_entity_preempt(&p->dl, &curr->dl)) && + (p->nr_cpus_allowed > 1)) { + int target = find_later_rq(p); + + if (target != -1) + cpu = target; + } + rcu_read_unlock(); + +out: + return cpu; +} + +static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p) +{ + /* + * Current can't be migrated, useless to reschedule, + * let's hope p can move out. + */ + if (rq->curr->nr_cpus_allowed == 1 || + cpudl_find(&rq->rd->cpudl, rq->curr, NULL) == -1) + return; + + /* + * p is migratable, so let's not schedule it and + * see if it is pushed or pulled somewhere else. + */ + if (p->nr_cpus_allowed != 1 && + cpudl_find(&rq->rd->cpudl, p, NULL) != -1) + return; + + resched_curr(rq); +} + +static int pull_dl_task(struct rq *this_rq); + +#endif /* CONFIG_SMP */ + +/* + * Only called when both the current and waking task are -deadline + * tasks. + */ +static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p, + int flags) +{ + if (dl_entity_preempt(&p->dl, &rq->curr->dl)) { + resched_curr(rq); + return; + } + +#ifdef CONFIG_SMP + /* + * In the unlikely case current and p have the same deadline + * let us try to decide what's the best thing to do... + */ + if ((p->dl.deadline == rq->curr->dl.deadline) && + !test_tsk_need_resched(rq->curr)) + check_preempt_equal_dl(rq, p); +#endif /* CONFIG_SMP */ +} + +#ifdef CONFIG_SCHED_HRTICK +static void start_hrtick_dl(struct rq *rq, struct task_struct *p) +{ + hrtick_start(rq, p->dl.runtime); +} +#endif + +static struct sched_dl_entity *pick_next_dl_entity(struct rq *rq, + struct dl_rq *dl_rq) +{ + struct rb_node *left = dl_rq->rb_leftmost; + + if (!left) + return NULL; + + return rb_entry(left, struct sched_dl_entity, rb_node); +} + +struct task_struct *pick_next_task_dl(struct rq *rq, struct task_struct *prev) +{ + struct sched_dl_entity *dl_se; + struct task_struct *p; + struct dl_rq *dl_rq; + + dl_rq = &rq->dl; + + if (need_pull_dl_task(rq, prev)) { + pull_dl_task(rq); + /* + * pull_rt_task() can drop (and re-acquire) rq->lock; this + * means a stop task can slip in, in which case we need to + * re-start task selection. + */ + if (rq->stop && task_on_rq_queued(rq->stop)) + return RETRY_TASK; + } + + /* + * When prev is DL, we may throttle it in put_prev_task(). + * So, we update time before we check for dl_nr_running. + */ + if (prev->sched_class == &dl_sched_class) + update_curr_dl(rq); + + if (unlikely(!dl_rq->dl_nr_running)) + return NULL; + + put_prev_task(rq, prev); + + dl_se = pick_next_dl_entity(rq, dl_rq); + BUG_ON(!dl_se); + + p = dl_task_of(dl_se); + p->se.exec_start = rq_clock_task(rq); + + /* Running task will never be pushed. */ + dequeue_pushable_dl_task(rq, p); + +#ifdef CONFIG_SCHED_HRTICK + if (hrtick_enabled(rq)) + start_hrtick_dl(rq, p); +#endif + + set_post_schedule(rq); + + return p; +} + +static void put_prev_task_dl(struct rq *rq, struct task_struct *p) +{ + update_curr_dl(rq); + + if (on_dl_rq(&p->dl) && p->nr_cpus_allowed > 1) + enqueue_pushable_dl_task(rq, p); +} + +static void task_tick_dl(struct rq *rq, struct task_struct *p, int queued) +{ + update_curr_dl(rq); + +#ifdef CONFIG_SCHED_HRTICK + if (hrtick_enabled(rq) && queued && p->dl.runtime > 0) + start_hrtick_dl(rq, p); +#endif +} + +static void task_fork_dl(struct task_struct *p) +{ + /* + * SCHED_DEADLINE tasks cannot fork and this is achieved through + * sched_fork() + */ +} + +static void task_dead_dl(struct task_struct *p) +{ + struct hrtimer *timer = &p->dl.dl_timer; + struct dl_bw *dl_b = dl_bw_of(task_cpu(p)); + + /* + * Since we are TASK_DEAD we won't slip out of the domain! + */ + raw_spin_lock_irq(&dl_b->lock); + dl_b->total_bw -= p->dl.dl_bw; + raw_spin_unlock_irq(&dl_b->lock); + + hrtimer_cancel(timer); +} + +static void set_curr_task_dl(struct rq *rq) +{ + struct task_struct *p = rq->curr; + + p->se.exec_start = rq_clock_task(rq); + + /* You can't push away the running task */ + dequeue_pushable_dl_task(rq, p); +} + +#ifdef CONFIG_SMP + +/* Only try algorithms three times */ +#define DL_MAX_TRIES 3 + +static int pick_dl_task(struct rq *rq, struct task_struct *p, int cpu) +{ + if (!task_running(rq, p) && + cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) + return 1; + return 0; +} + +/* Returns the second earliest -deadline task, NULL otherwise */ +static struct task_struct *pick_next_earliest_dl_task(struct rq *rq, int cpu) +{ + struct rb_node *next_node = rq->dl.rb_leftmost; + struct sched_dl_entity *dl_se; + struct task_struct *p = NULL; + +next_node: + next_node = rb_next(next_node); + if (next_node) { + dl_se = rb_entry(next_node, struct sched_dl_entity, rb_node); + p = dl_task_of(dl_se); + + if (pick_dl_task(rq, p, cpu)) + return p; + + goto next_node; + } + + return NULL; +} + +static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask_dl); + +static int find_later_rq(struct task_struct *task) +{ + struct sched_domain *sd; + struct cpumask *later_mask = this_cpu_ptr(local_cpu_mask_dl); + int this_cpu = smp_processor_id(); + int best_cpu, cpu = task_cpu(task); + + /* Make sure the mask is initialized first */ + if (unlikely(!later_mask)) + return -1; + + if (task->nr_cpus_allowed == 1) + return -1; + + /* + * We have to consider system topology and task affinity + * first, then we can look for a suitable cpu. + */ + cpumask_copy(later_mask, task_rq(task)->rd->span); + cpumask_and(later_mask, later_mask, cpu_active_mask); + cpumask_and(later_mask, later_mask, &task->cpus_allowed); + best_cpu = cpudl_find(&task_rq(task)->rd->cpudl, + task, later_mask); + if (best_cpu == -1) + return -1; + + /* + * If we are here, some target has been found, + * the most suitable of which is cached in best_cpu. + * This is, among the runqueues where the current tasks + * have later deadlines than the task's one, the rq + * with the latest possible one. + * + * Now we check how well this matches with task's + * affinity and system topology. + * + * The last cpu where the task run is our first + * guess, since it is most likely cache-hot there. + */ + if (cpumask_test_cpu(cpu, later_mask)) + return cpu; + /* + * Check if this_cpu is to be skipped (i.e., it is + * not in the mask) or not. + */ + if (!cpumask_test_cpu(this_cpu, later_mask)) + this_cpu = -1; + + rcu_read_lock(); + for_each_domain(cpu, sd) { + if (sd->flags & SD_WAKE_AFFINE) { + + /* + * If possible, preempting this_cpu is + * cheaper than migrating. + */ + if (this_cpu != -1 && + cpumask_test_cpu(this_cpu, sched_domain_span(sd))) { + rcu_read_unlock(); + return this_cpu; + } + + /* + * Last chance: if best_cpu is valid and is + * in the mask, that becomes our choice. + */ + if (best_cpu < nr_cpu_ids && + cpumask_test_cpu(best_cpu, sched_domain_span(sd))) { + rcu_read_unlock(); + return best_cpu; + } + } + } + rcu_read_unlock(); + + /* + * At this point, all our guesses failed, we just return + * 'something', and let the caller sort the things out. + */ + if (this_cpu != -1) + return this_cpu; + + cpu = cpumask_any(later_mask); + if (cpu < nr_cpu_ids) + return cpu; + + return -1; +} + +/* Locks the rq it finds */ +static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq) +{ + struct rq *later_rq = NULL; + int tries; + int cpu; + + for (tries = 0; tries < DL_MAX_TRIES; tries++) { + cpu = find_later_rq(task); + + if ((cpu == -1) || (cpu == rq->cpu)) + break; + + later_rq = cpu_rq(cpu); + + /* Retry if something changed. */ + if (double_lock_balance(rq, later_rq)) { + if (unlikely(task_rq(task) != rq || + !cpumask_test_cpu(later_rq->cpu, + &task->cpus_allowed) || + task_running(rq, task) || + !task_on_rq_queued(task))) { + double_unlock_balance(rq, later_rq); + later_rq = NULL; + break; + } + } + + /* + * If the rq we found has no -deadline task, or + * its earliest one has a later deadline than our + * task, the rq is a good one. + */ + if (!later_rq->dl.dl_nr_running || + dl_time_before(task->dl.deadline, + later_rq->dl.earliest_dl.curr)) + break; + + /* Otherwise we try again. */ + double_unlock_balance(rq, later_rq); + later_rq = NULL; + } + + return later_rq; +} + +static struct task_struct *pick_next_pushable_dl_task(struct rq *rq) +{ + struct task_struct *p; + + if (!has_pushable_dl_tasks(rq)) + return NULL; + + p = rb_entry(rq->dl.pushable_dl_tasks_leftmost, + struct task_struct, pushable_dl_tasks); + + BUG_ON(rq->cpu != task_cpu(p)); + BUG_ON(task_current(rq, p)); + BUG_ON(p->nr_cpus_allowed <= 1); + + BUG_ON(!task_on_rq_queued(p)); + BUG_ON(!dl_task(p)); + + return p; +} + +/* + * See if the non running -deadline tasks on this rq + * can be sent to some other CPU where they can preempt + * and start executing. + */ +static int push_dl_task(struct rq *rq) +{ + struct task_struct *next_task; + struct rq *later_rq; + + if (!rq->dl.overloaded) + return 0; + + next_task = pick_next_pushable_dl_task(rq); + if (!next_task) + return 0; + +retry: + if (unlikely(next_task == rq->curr)) { + WARN_ON(1); + return 0; + } + + /* + * If next_task preempts rq->curr, and rq->curr + * can move away, it makes sense to just reschedule + * without going further in pushing next_task. + */ + if (dl_task(rq->curr) && + dl_time_before(next_task->dl.deadline, rq->curr->dl.deadline) && + rq->curr->nr_cpus_allowed > 1) { + resched_curr(rq); + return 0; + } + + /* We might release rq lock */ + get_task_struct(next_task); + + /* Will lock the rq it'll find */ + later_rq = find_lock_later_rq(next_task, rq); + if (!later_rq) { + struct task_struct *task; + + /* + * We must check all this again, since + * find_lock_later_rq releases rq->lock and it is + * then possible that next_task has migrated. + */ + task = pick_next_pushable_dl_task(rq); + if (task_cpu(next_task) == rq->cpu && task == next_task) { + /* + * The task is still there. We don't try + * again, some other cpu will pull it when ready. + */ + dequeue_pushable_dl_task(rq, next_task); + goto out; + } + + if (!task) + /* No more tasks */ + goto out; + + put_task_struct(next_task); + next_task = task; + goto retry; + } + + deactivate_task(rq, next_task, 0); + set_task_cpu(next_task, later_rq->cpu); + activate_task(later_rq, next_task, 0); + + resched_curr(later_rq); + + double_unlock_balance(rq, later_rq); + +out: + put_task_struct(next_task); + + return 1; +} + +static void push_dl_tasks(struct rq *rq) +{ + /* Terminates as it moves a -deadline task */ + while (push_dl_task(rq)) + ; +} + +static int pull_dl_task(struct rq *this_rq) +{ + int this_cpu = this_rq->cpu, ret = 0, cpu; + struct task_struct *p; + struct rq *src_rq; + u64 dmin = LONG_MAX; + + if (likely(!dl_overloaded(this_rq))) + return 0; + + /* + * Match the barrier from dl_set_overloaded; this guarantees that if we + * see overloaded we must also see the dlo_mask bit. + */ + smp_rmb(); + + for_each_cpu(cpu, this_rq->rd->dlo_mask) { + if (this_cpu == cpu) + continue; + + src_rq = cpu_rq(cpu); + + /* + * It looks racy, abd it is! However, as in sched_rt.c, + * we are fine with this. + */ + if (this_rq->dl.dl_nr_running && + dl_time_before(this_rq->dl.earliest_dl.curr, + src_rq->dl.earliest_dl.next)) + continue; + + /* Might drop this_rq->lock */ + double_lock_balance(this_rq, src_rq); + + /* + * If there are no more pullable tasks on the + * rq, we're done with it. + */ + if (src_rq->dl.dl_nr_running <= 1) + goto skip; + + p = pick_next_earliest_dl_task(src_rq, this_cpu); + + /* + * We found a task to be pulled if: + * - it preempts our current (if there's one), + * - it will preempt the last one we pulled (if any). + */ + if (p && dl_time_before(p->dl.deadline, dmin) && + (!this_rq->dl.dl_nr_running || + dl_time_before(p->dl.deadline, + this_rq->dl.earliest_dl.curr))) { + WARN_ON(p == src_rq->curr); + WARN_ON(!task_on_rq_queued(p)); + + /* + * Then we pull iff p has actually an earlier + * deadline than the current task of its runqueue. + */ + if (dl_time_before(p->dl.deadline, + src_rq->curr->dl.deadline)) + goto skip; + + ret = 1; + + deactivate_task(src_rq, p, 0); + set_task_cpu(p, this_cpu); + activate_task(this_rq, p, 0); + dmin = p->dl.deadline; + + /* Is there any other task even earlier? */ + } +skip: + double_unlock_balance(this_rq, src_rq); + } + + return ret; +} + +static void post_schedule_dl(struct rq *rq) +{ + push_dl_tasks(rq); +} + +/* + * Since the task is not running and a reschedule is not going to happen + * anytime soon on its runqueue, we try pushing it away now. + */ +static void task_woken_dl(struct rq *rq, struct task_struct *p) +{ + if (!task_running(rq, p) && + !test_tsk_need_resched(rq->curr) && + has_pushable_dl_tasks(rq) && + p->nr_cpus_allowed > 1 && + dl_task(rq->curr) && + (rq->curr->nr_cpus_allowed < 2 || + dl_entity_preempt(&rq->curr->dl, &p->dl))) { + push_dl_tasks(rq); + } +} + +static void set_cpus_allowed_dl(struct task_struct *p, + const struct cpumask *new_mask) +{ + struct rq *rq; + int weight; + + BUG_ON(!dl_task(p)); + + /* + * Update only if the task is actually running (i.e., + * it is on the rq AND it is not throttled). + */ + if (!on_dl_rq(&p->dl)) + return; + + weight = cpumask_weight(new_mask); + + /* + * Only update if the process changes its state from whether it + * can migrate or not. + */ + if ((p->nr_cpus_allowed > 1) == (weight > 1)) + return; + + rq = task_rq(p); + + /* + * The process used to be able to migrate OR it can now migrate + */ + if (weight <= 1) { + if (!task_current(rq, p)) + dequeue_pushable_dl_task(rq, p); + BUG_ON(!rq->dl.dl_nr_migratory); + rq->dl.dl_nr_migratory--; + } else { + if (!task_current(rq, p)) + enqueue_pushable_dl_task(rq, p); + rq->dl.dl_nr_migratory++; + } + + update_dl_migration(&rq->dl); +} + +/* Assumes rq->lock is held */ +static void rq_online_dl(struct rq *rq) +{ + if (rq->dl.overloaded) + dl_set_overload(rq); + + if (rq->dl.dl_nr_running > 0) + cpudl_set(&rq->rd->cpudl, rq->cpu, rq->dl.earliest_dl.curr, 1); +} + +/* Assumes rq->lock is held */ +static void rq_offline_dl(struct rq *rq) +{ + if (rq->dl.overloaded) + dl_clear_overload(rq); + + cpudl_set(&rq->rd->cpudl, rq->cpu, 0, 0); +} + +void init_sched_dl_class(void) +{ + unsigned int i; + + for_each_possible_cpu(i) + zalloc_cpumask_var_node(&per_cpu(local_cpu_mask_dl, i), + GFP_KERNEL, cpu_to_node(i)); +} + +#endif /* CONFIG_SMP */ + +static void switched_from_dl(struct rq *rq, struct task_struct *p) +{ + if (hrtimer_active(&p->dl.dl_timer) && !dl_policy(p->policy)) + hrtimer_try_to_cancel(&p->dl.dl_timer); + + __dl_clear_params(p); + +#ifdef CONFIG_SMP + /* + * Since this might be the only -deadline task on the rq, + * this is the right place to try to pull some other one + * from an overloaded cpu, if any. + */ + if (!rq->dl.dl_nr_running) + pull_dl_task(rq); +#endif +} + +/* + * When switching to -deadline, we may overload the rq, then + * we try to push someone off, if possible. + */ +static void switched_to_dl(struct rq *rq, struct task_struct *p) +{ + int check_resched = 1; + + /* + * If p is throttled, don't consider the possibility + * of preempting rq->curr, the check will be done right + * after its runtime will get replenished. + */ + if (unlikely(p->dl.dl_throttled)) + return; + + if (task_on_rq_queued(p) && rq->curr != p) { +#ifdef CONFIG_SMP + if (rq->dl.overloaded && push_dl_task(rq) && rq != task_rq(p)) + /* Only reschedule if pushing failed */ + check_resched = 0; +#endif /* CONFIG_SMP */ + if (check_resched) { + if (dl_task(rq->curr)) + check_preempt_curr_dl(rq, p, 0); + else + resched_curr(rq); + } + } +} + +/* + * If the scheduling parameters of a -deadline task changed, + * a push or pull operation might be needed. + */ +static void prio_changed_dl(struct rq *rq, struct task_struct *p, + int oldprio) +{ + if (task_on_rq_queued(p) || rq->curr == p) { +#ifdef CONFIG_SMP + /* + * This might be too much, but unfortunately + * we don't have the old deadline value, and + * we can't argue if the task is increasing + * or lowering its prio, so... + */ + if (!rq->dl.overloaded) + pull_dl_task(rq); + + /* + * If we now have a earlier deadline task than p, + * then reschedule, provided p is still on this + * runqueue. + */ + if (dl_time_before(rq->dl.earliest_dl.curr, p->dl.deadline) && + rq->curr == p) + resched_curr(rq); +#else + /* + * Again, we don't know if p has a earlier + * or later deadline, so let's blindly set a + * (maybe not needed) rescheduling point. + */ + resched_curr(rq); +#endif /* CONFIG_SMP */ + } else + switched_to_dl(rq, p); +} + +const struct sched_class dl_sched_class = { + .next = &rt_sched_class, + .enqueue_task = enqueue_task_dl, + .dequeue_task = dequeue_task_dl, + .yield_task = yield_task_dl, + + .check_preempt_curr = check_preempt_curr_dl, + + .pick_next_task = pick_next_task_dl, + .put_prev_task = put_prev_task_dl, + +#ifdef CONFIG_SMP + .select_task_rq = select_task_rq_dl, + .set_cpus_allowed = set_cpus_allowed_dl, + .rq_online = rq_online_dl, + .rq_offline = rq_offline_dl, + .post_schedule = post_schedule_dl, + .task_woken = task_woken_dl, +#endif + + .set_curr_task = set_curr_task_dl, + .task_tick = task_tick_dl, + .task_fork = task_fork_dl, + .task_dead = task_dead_dl, + + .prio_changed = prio_changed_dl, + .switched_from = switched_from_dl, + .switched_to = switched_to_dl, + + .update_curr = update_curr_dl, +}; diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index a4e8ef3aca901..14c0967e7218b 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "sched.h" @@ -70,7 +71,7 @@ static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group if (!se) { struct sched_avg *avg = &cpu_rq(cpu)->avg; P(avg->runnable_avg_sum); - P(avg->runnable_avg_period); + P(avg->avg_period); return; } @@ -93,7 +94,7 @@ static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group P(se->load.weight); #ifdef CONFIG_SMP P(se->avg.runnable_avg_sum); - P(se->avg.runnable_avg_period); + P(se->avg.avg_period); P(se->avg.load_avg_contrib); P(se->avg.decay_count); #endif @@ -124,7 +125,7 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) SEQ_printf(m, " "); SEQ_printf(m, "%15s %5d %9Ld.%06ld %9Ld %5d ", - p->comm, p->pid, + p->comm, task_pid_nr(p), SPLIT_NS(p->se.vruntime), (long long)(p->nvcsw + p->nivcsw), p->prio); @@ -137,6 +138,9 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) SEQ_printf(m, "%15Ld %15Ld %15Ld.%06ld %15Ld.%06ld %15Ld.%06ld", 0LL, 0LL, 0LL, 0L, 0LL, 0L, 0LL, 0L); #endif +#ifdef CONFIG_NUMA_BALANCING + SEQ_printf(m, " %d", task_node(p)); +#endif #ifdef CONFIG_CGROUP_SCHED SEQ_printf(m, " %s", task_group_path(task_group(p))); #endif @@ -147,7 +151,6 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) { struct task_struct *g, *p; - unsigned long flags; SEQ_printf(m, "\nrunnable tasks:\n" @@ -156,16 +159,14 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) "------------------------------------------------------" "----------------------------------------------------\n"); - read_lock_irqsave(&tasklist_lock, flags); - - do_each_thread(g, p) { - if (!p->on_rq || task_cpu(p) != rq_cpu) + rcu_read_lock(); + for_each_process_thread(g, p) { + if (task_cpu(p) != rq_cpu) continue; print_task(m, rq, p); - } while_each_thread(g, p); - - read_unlock_irqrestore(&tasklist_lock, flags); + } + rcu_read_unlock(); } void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) @@ -209,29 +210,25 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) cfs_rq->nr_spread_over); SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running); SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight); -#ifdef CONFIG_CFS_BANDWIDTH - SEQ_printf(m, " .%-30s: %d\n", "tg->cfs_bandwidth.timer_active", - cfs_rq->tg->cfs_bandwidth.timer_active); - SEQ_printf(m, " .%-30s: %d\n", "throttled", - cfs_rq->throttled); - SEQ_printf(m, " .%-30s: %d\n", "throttle_count", - cfs_rq->throttle_count); -#endif #ifdef CONFIG_FAIR_GROUP_SCHED #ifdef CONFIG_SMP - SEQ_printf(m, " .%-30s: %lld\n", "runnable_load_avg", + SEQ_printf(m, " .%-30s: %ld\n", "runnable_load_avg", cfs_rq->runnable_load_avg); - SEQ_printf(m, " .%-30s: %lld\n", "blocked_load_avg", + SEQ_printf(m, " .%-30s: %ld\n", "blocked_load_avg", cfs_rq->blocked_load_avg); - SEQ_printf(m, " .%-30s: %lld\n", "tg_load_avg", - (unsigned long long)atomic64_read(&cfs_rq->tg->load_avg)); - SEQ_printf(m, " .%-30s: %lld\n", "tg_load_contrib", + SEQ_printf(m, " .%-30s: %ld\n", "utilization_load_avg", + cfs_rq->utilization_load_avg); +#ifdef CONFIG_FAIR_GROUP_SCHED + SEQ_printf(m, " .%-30s: %ld\n", "tg_load_contrib", cfs_rq->tg_load_contrib); SEQ_printf(m, " .%-30s: %d\n", "tg_runnable_contrib", cfs_rq->tg_runnable_contrib); + SEQ_printf(m, " .%-30s: %ld\n", "tg_load_avg", + atomic_long_read(&cfs_rq->tg->load_avg)); SEQ_printf(m, " .%-30s: %d\n", "tg->runnable_avg", atomic_read(&cfs_rq->tg->runnable_avg)); #endif +#endif #ifdef CONFIG_CFS_BANDWIDTH SEQ_printf(m, " .%-30s: %d\n", "tg->cfs_bandwidth.timer_active", cfs_rq->tg->cfs_bandwidth.timer_active); @@ -239,18 +236,9 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) cfs_rq->throttled); SEQ_printf(m, " .%-30s: %d\n", "throttle_count", cfs_rq->throttle_count); - SEQ_printf(m, " .%-30s: %d\n", "runtime_enabled", - cfs_rq->runtime_enabled); -#ifdef CONFIG_SCHED_HMP - SEQ_printf(m, " .%-30s: %d\n", "nr_big_tasks", - cfs_rq->hmp_stats.nr_big_tasks); - SEQ_printf(m, " .%-30s: %d\n", "nr_small_tasks", - cfs_rq->hmp_stats.nr_small_tasks); - SEQ_printf(m, " .%-30s: %llu\n", "cumulative_runnable_avg", - cfs_rq->hmp_stats.cumulative_runnable_avg); -#endif #endif +#ifdef CONFIG_FAIR_GROUP_SCHED print_cfs_group_stats(m, cpu, cfs_rq->tg); #endif } @@ -313,32 +301,13 @@ do { \ P(nr_load_updates); P(nr_uninterruptible); PN(next_balance); - P(curr->pid); + SEQ_printf(m, " .%-30s: %ld\n", "curr->pid", (long)(task_pid_nr(rq->curr))); PN(clock); P(cpu_load[0]); P(cpu_load[1]); P(cpu_load[2]); P(cpu_load[3]); P(cpu_load[4]); -#ifdef CONFIG_SMP - P(cpu_power); -#endif -#ifdef CONFIG_SCHED_HMP - P(mostly_idle_load); - P(mostly_idle_nr_run); - P(load_scale_factor); - P(capacity); - P(max_possible_capacity); - P(efficiency); - P(cur_freq); - P(max_freq); -#endif -#ifdef CONFIG_SCHED_HMP - P(hmp_stats.nr_big_tasks); - P(hmp_stats.nr_small_tasks); - SEQ_printf(m, " .%-30s: %llu\n", "hmp_stats.cumulative_runnable_avg", - rq->hmp_stats.cumulative_runnable_avg); -#endif #undef P #undef PN @@ -352,6 +321,7 @@ do { \ P(sched_goidle); #ifdef CONFIG_SMP P64(avg_idle); + P64(max_idle_balance_cost); #endif P(ttwu_count); @@ -364,9 +334,7 @@ do { \ print_cfs_stats(m, cpu); print_rt_stats(m, cpu); - rcu_read_lock(); print_rq(m, rq, cpu); - rcu_read_unlock(); spin_unlock_irqrestore(&sched_debug_lock, flags); SEQ_printf(m, "\n"); } @@ -388,7 +356,7 @@ static void sched_debug_header(struct seq_file *m) cpu_clk = local_clock(); local_irq_restore(flags); - SEQ_printf(m, "Sched Debug Version: v0.10, %s %.*s\n", + SEQ_printf(m, "Sched Debug Version: v0.11, %s %.*s\n", init_utsname()->release, (int)strcspn(init_utsname()->version, " "), init_utsname()->version); @@ -402,7 +370,7 @@ static void sched_debug_header(struct seq_file *m) PN(cpu_clk); P(jiffies); #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK - P(sched_clock_stable); + P(sched_clock_stable()); #endif #undef PN #undef P @@ -419,17 +387,6 @@ static void sched_debug_header(struct seq_file *m) PN(sysctl_sched_wakeup_granularity); P(sysctl_sched_child_runs_first); P(sysctl_sched_features); -#ifdef CONFIG_SCHED_HMP - P(sched_small_task); - P(sched_upmigrate); - P(sched_downmigrate); - P(sched_init_task_load_windows); - P(sched_init_task_load_pelt); - P(min_capacity); - P(max_capacity); - P(sched_use_pelt); - P(sched_ravg_window); -#endif #undef PN #undef P @@ -452,7 +409,6 @@ static int sched_debug_show(struct seq_file *m, void *v) return 0; } -#ifdef CONFIG_SYSRQ_SCHED_DEBUG void sysrq_sched_debug_show(void) { int cpu; @@ -462,7 +418,6 @@ void sysrq_sched_debug_show(void) print_cpu(NULL, cpu); } -#endif /* * This itererator needs some explanation. @@ -544,25 +499,73 @@ static int __init init_sched_debug_procfs(void) __initcall(init_sched_debug_procfs); +#define __P(F) \ + SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)F) +#define P(F) \ + SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)p->F) +#define __PN(F) \ + SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F)) +#define PN(F) \ + SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F)) + + +static void sched_show_numa(struct task_struct *p, struct seq_file *m) +{ +#ifdef CONFIG_NUMA_BALANCING + struct mempolicy *pol; + int node, i; + + if (p->mm) + P(mm->numa_scan_seq); + + task_lock(p); + pol = p->mempolicy; + if (pol && !(pol->flags & MPOL_F_MORON)) + pol = NULL; + mpol_get(pol); + task_unlock(p); + + SEQ_printf(m, "numa_migrations, %ld\n", xchg(&p->numa_pages_migrated, 0)); + + for_each_online_node(node) { + for (i = 0; i < 2; i++) { + unsigned long nr_faults = -1; + int cpu_current, home_node; + + if (p->numa_faults_memory) + nr_faults = p->numa_faults_memory[2*node + i]; + + cpu_current = !i ? (task_node(p) == node) : + (pol && node_isset(node, pol->v.nodes)); + + home_node = (p->numa_preferred_nid == node); + + SEQ_printf(m, "numa_faults_memory, %d, %d, %d, %d, %ld\n", + i, node, cpu_current, home_node, nr_faults); + } + } + + mpol_put(pol); +#endif +} + void proc_sched_show_task(struct task_struct *p, struct seq_file *m) { unsigned long nr_switches; - unsigned int load_avg; - - load_avg = pct_task_load(p); - SEQ_printf(m, "%s (%d, #threads: %d)\n", p->comm, p->pid, + SEQ_printf(m, "%s (%d, #threads: %d)\n", p->comm, task_pid_nr(p), get_nr_threads(p)); SEQ_printf(m, - "---------------------------------------------------------\n"); + "---------------------------------------------------------" + "----------\n"); #define __P(F) \ - SEQ_printf(m, "%-35s:%21Ld\n", #F, (long long)F) + SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)F) #define P(F) \ - SEQ_printf(m, "%-35s:%21Ld\n", #F, (long long)p->F) + SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)p->F) #define __PN(F) \ - SEQ_printf(m, "%-35s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F)) + SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F)) #define PN(F) \ - SEQ_printf(m, "%-35s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F)) + SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F)) PN(se.exec_start); PN(se.vruntime); @@ -599,16 +602,6 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) P(se.statistics.nr_wakeups_passive); P(se.statistics.nr_wakeups_idle); -#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED) - __P(load_avg); -#ifdef CONFIG_SCHED_HMP - P(ravg.demand); - P(se.avg.runnable_avg_sum_scaled); -#endif - P(se.avg.runnable_avg_sum); - P(se.avg.runnable_avg_period); -#endif - { u64 avg_atom, avg_per_cpu; @@ -631,12 +624,20 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) } #endif __P(nr_switches); - SEQ_printf(m, "%-35s:%21Ld\n", + SEQ_printf(m, "%-45s:%21Ld\n", "nr_voluntary_switches", (long long)p->nvcsw); - SEQ_printf(m, "%-35s:%21Ld\n", + SEQ_printf(m, "%-45s:%21Ld\n", "nr_involuntary_switches", (long long)p->nivcsw); P(se.load.weight); +#ifdef CONFIG_SMP + P(se.avg.runnable_avg_sum); + P(se.avg.running_avg_sum); + P(se.avg.avg_period); + P(se.avg.load_avg_contrib); + P(se.avg.utilization_avg_contrib); + P(se.avg.decay_count); +#endif P(policy); P(prio); #undef PN @@ -650,9 +651,11 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) t0 = cpu_clock(this_cpu); t1 = cpu_clock(this_cpu); - SEQ_printf(m, "%-35s:%21Ld\n", + SEQ_printf(m, "%-45s:%21Ld\n", "clock-delta", (long long)(t1-t0)); } + + sched_show_numa(p, m); } void proc_sched_set_task(struct task_struct *p) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 8942198ac1b85..efc7128798854 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -33,6 +34,14 @@ #include #include "sched.h" +#include "tune.h" + +#ifndef TJK_HMP +// TJK: resolve undefined sysctl symbols +unsigned int __read_mostly sysctl_sched_wake_to_idle = 0; +__read_mostly unsigned int sysctl_sched_wakeup_load_threshold = 110; +#endif + /* * Targeted preemption latency for CPU-bound tasks: @@ -79,32 +88,6 @@ static unsigned int sched_nr_latency = 8; */ unsigned int sysctl_sched_child_runs_first __read_mostly; -#ifdef VENDOR_EDIT -enum thermal_aware_scheduling { - SCHED_TA_DISABLE, - SCHED_TA_THERMAL_ONLY, - SCHED_TA_ALWAYS_ON, -}; -/* - * Thermal-aware scheduling, prefer running on small cluster - * when task loading on big cluster is below certain threshold - * - * Options: - * SCHED_TA_DISABLE - Disable TA scheduling - * SCHED_TA_THERMAL_ONLY - Only enable when under thermal constraint - * SCHED_TA_ALWAYS_ON - Always enable TA scheduling - */ -unsigned int __read_mostly sysctl_thermal_aware_scheduling = SCHED_TA_THERMAL_ONLY; -#endif - -/* - * Controls whether, when SD_SHARE_PKG_RESOURCES is on, if all - * tasks go to idle CPUs when woken. If this is off, note that the - * per-task flag PF_WAKE_UP_IDLE can still cause a task to go to an - * idle CPU upon being woken. - */ -unsigned int __read_mostly sysctl_sched_wake_to_idle; - /* * SCHED_OTHER wake-up granularity. * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) @@ -139,6 +122,24 @@ unsigned int __read_mostly sysctl_sched_shares_window = 10000000UL; unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; #endif +static inline void update_load_add(struct load_weight *lw, unsigned long inc) +{ + lw->weight += inc; + lw->inv_weight = 0; +} + +static inline void update_load_sub(struct load_weight *lw, unsigned long dec) +{ + lw->weight -= dec; + lw->inv_weight = 0; +} + +static inline void update_load_set(struct load_weight *lw, unsigned long w) +{ + lw->weight = w; + lw->inv_weight = 0; +} + /* * Increase the granularity value when there are more CPUs, * because with more CPUs the 'effective latency' as visible @@ -186,64 +187,78 @@ void sched_init_granularity(void) update_sysctl(); } -#if BITS_PER_LONG == 32 -# define WMULT_CONST (~0UL) -#else -# define WMULT_CONST (1UL << 32) -#endif - +#define WMULT_CONST (~0U) #define WMULT_SHIFT 32 -/* - * Shift right and round: - */ -#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y)) +static void __update_inv_weight(struct load_weight *lw) +{ + unsigned long w; + + if (likely(lw->inv_weight)) + return; + + w = scale_load_down(lw->weight); + + if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) + lw->inv_weight = 1; + else if (unlikely(!w)) + lw->inv_weight = WMULT_CONST; + else + lw->inv_weight = WMULT_CONST / w; +} + +static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift) +{ + u32 ah, al; + u64 ret; + + al = a; + ah = a >> 32; + + ret = ((u64)al * mul) >> shift; + if (ah) + ret += ((u64)ah * mul) << (32 - shift); + + return ret; +} /* - * delta *= weight / lw + * delta_exec * weight / lw.weight + * OR + * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT + * + * Either weight := NICE_0_LOAD and lw \e prio_to_wmult[], in which case + * we're guaranteed shift stays positive because inv_weight is guaranteed to + * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22. + * + * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus + * weight/lw.weight <= 1, and therefore our shift will also be positive. */ -static unsigned long -calc_delta_mine(unsigned long delta_exec, unsigned long weight, - struct load_weight *lw) +static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) { - u64 tmp; - - /* - * weight can be less than 2^SCHED_LOAD_RESOLUTION for task group sched - * entities since MIN_SHARES = 2. Treat weight as 1 if less than - * 2^SCHED_LOAD_RESOLUTION. - */ - if (likely(weight > (1UL << SCHED_LOAD_RESOLUTION))) - tmp = (u64)delta_exec * scale_load_down(weight); - else - tmp = (u64)delta_exec; + u64 fact = scale_load_down(weight); + int shift = WMULT_SHIFT; - if (!lw->inv_weight) { - unsigned long w = scale_load_down(lw->weight); + __update_inv_weight(lw); - if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) - lw->inv_weight = 1; - else if (unlikely(!w)) - lw->inv_weight = WMULT_CONST; - else - lw->inv_weight = WMULT_CONST / w; + if (unlikely(fact >> 32)) { + while (fact >> 32) { + fact >>= 1; + shift--; + } } - /* - * Check whether we'd overflow the 64-bit multiplication: - */ - if (unlikely(tmp > WMULT_CONST)) - tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight, - WMULT_SHIFT/2); - else - tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT); + /* hint to use a 32x32->64 mul */ + fact = (u64)(u32)fact * lw->inv_weight; - return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX); + while (fact >> 32) { + fact >>= 1; + shift--; + } + + return mul_u64_u32_shr(delta_exec, fact, shift); } -#ifdef CONFIG_SMP -static int active_load_balance_cpu_stop(void *data); -#endif const struct sched_class fair_sched_class; @@ -331,13 +346,13 @@ static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) list_for_each_entry_rcu(cfs_rq, &rq->leaf_cfs_rq_list, leaf_cfs_rq_list) /* Do the two (enqueued) entities belong to the same group ? */ -static inline int +static inline struct cfs_rq * is_same_group(struct sched_entity *se, struct sched_entity *pse) { if (se->cfs_rq == pse->cfs_rq) - return 1; + return se->cfs_rq; - return 0; + return NULL; } static inline struct sched_entity *parent_entity(struct sched_entity *se) @@ -345,17 +360,6 @@ static inline struct sched_entity *parent_entity(struct sched_entity *se) return se->parent; } -/* return depth at which a sched entity is present in the hierarchy */ -static inline int depth_se(struct sched_entity *se) -{ - int depth = 0; - - for_each_sched_entity(se) - depth++; - - return depth; -} - static void find_matching_se(struct sched_entity **se, struct sched_entity **pse) { @@ -369,8 +373,8 @@ find_matching_se(struct sched_entity **se, struct sched_entity **pse) */ /* First walk up until both entities are at same depth */ - se_depth = depth_se(*se); - pse_depth = depth_se(*pse); + se_depth = (*se)->depth; + pse_depth = (*pse)->depth; while (se_depth > pse_depth) { se_depth--; @@ -435,12 +439,6 @@ static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) #define for_each_leaf_cfs_rq(rq, cfs_rq) \ for (cfs_rq = &rq->cfs; cfs_rq; cfs_rq = NULL) -static inline int -is_same_group(struct sched_entity *se, struct sched_entity *pse) -{ - return 1; -} - static inline struct sched_entity *parent_entity(struct sched_entity *se) { return NULL; @@ -454,7 +452,7 @@ find_matching_se(struct sched_entity **se, struct sched_entity **pse) #endif /* CONFIG_FAIR_GROUP_SCHED */ static __always_inline -void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, unsigned long delta_exec); +void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); /************************************************************** * Scheduling class tree data structure manipulation methods: @@ -623,11 +621,10 @@ int sched_proc_update_handler(struct ctl_table *table, int write, /* * delta /= w */ -static inline unsigned long -calc_delta_fair(unsigned long delta, struct sched_entity *se) +static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) { if (unlikely(se->load.weight != NICE_0_LOAD)) - delta = calc_delta_mine(delta, NICE_0_LOAD, &se->load); + delta = __calc_delta(delta, NICE_0_LOAD, &se->load); return delta; } @@ -676,7 +673,7 @@ static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) update_load_add(&lw, se->load.weight); load = &lw; } - slice = calc_delta_mine(slice, se->load.weight, load); + slice = __calc_delta(slice, se->load.weight, load); } return slice; } @@ -691,48 +688,57 @@ static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se) return calc_delta_fair(sched_slice(cfs_rq, se), se); } -/* - * Update the current task's runtime statistics. Skip current tasks that - * are not in our scheduling class. - */ -static inline void -__update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr, - unsigned long delta_exec) -{ - unsigned long delta_exec_weighted; +#ifdef CONFIG_SMP +static int select_idle_sibling(struct task_struct *p, int cpu); +static unsigned long task_h_load(struct task_struct *p); - schedstat_set(curr->statistics.exec_max, - max((u64)delta_exec, curr->statistics.exec_max)); +static inline void __update_task_entity_contrib(struct sched_entity *se); +static inline void __update_task_entity_utilization(struct sched_entity *se); - curr->sum_exec_runtime += delta_exec; - schedstat_add(cfs_rq, exec_clock, delta_exec); - delta_exec_weighted = calc_delta_fair(delta_exec, curr); +/* Give new task start runnable values to heavy its load in infant time */ +void init_task_runnable_average(struct task_struct *p) +{ + u32 start_load = sysctl_sched_latency >> 10; - curr->vruntime += delta_exec_weighted; - update_min_vruntime(cfs_rq); + p->se.avg.decay_count = 0; + p->se.avg.runnable_avg_sum = p->se.avg.running_avg_sum = start_load; + p->se.avg.avg_period = start_load; + __update_task_entity_contrib(&p->se); + __update_task_entity_utilization(&p->se); +} +#else +void init_task_runnable_average(struct task_struct *p) +{ } +#endif +/* + * Update the current task's runtime statistics. + */ static void update_curr(struct cfs_rq *cfs_rq) { struct sched_entity *curr = cfs_rq->curr; - u64 now = rq_of(cfs_rq)->clock_task; - unsigned long delta_exec; + u64 now = rq_clock_task(rq_of(cfs_rq)); + u64 delta_exec; if (unlikely(!curr)) return; - /* - * Get the amount of time the current task was running - * since the last time we changed load (this cannot - * overflow on 32 bits): - */ - delta_exec = (unsigned long)(now - curr->exec_start); - if (!delta_exec) + delta_exec = now - curr->exec_start; + if (unlikely((s64)delta_exec <= 0)) return; - __update_curr(cfs_rq, curr, delta_exec); curr->exec_start = now; + schedstat_set(curr->statistics.exec_max, + max(delta_exec, curr->statistics.exec_max)); + + curr->sum_exec_runtime += delta_exec; + schedstat_add(cfs_rq, exec_clock, delta_exec); + + curr->vruntime += calc_delta_fair(delta_exec, curr); + update_min_vruntime(cfs_rq); + if (entity_is_task(curr)) { struct task_struct *curtask = task_of(curr); @@ -744,10 +750,15 @@ static void update_curr(struct cfs_rq *cfs_rq) account_cfs_rq_runtime(cfs_rq, delta_exec); } +static void update_curr_fair(struct rq *rq) +{ + update_curr(cfs_rq_of(&rq->curr->se)); +} + static inline void update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se) { - schedstat_set(se->statistics.wait_start, rq_of(cfs_rq)->clock); + schedstat_set(se->statistics.wait_start, rq_clock(rq_of(cfs_rq))); } /* @@ -767,14 +778,14 @@ static void update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) { schedstat_set(se->statistics.wait_max, max(se->statistics.wait_max, - rq_of(cfs_rq)->clock - se->statistics.wait_start)); + rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start)); schedstat_set(se->statistics.wait_count, se->statistics.wait_count + 1); schedstat_set(se->statistics.wait_sum, se->statistics.wait_sum + - rq_of(cfs_rq)->clock - se->statistics.wait_start); + rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start); #ifdef CONFIG_SCHEDSTATS if (entity_is_task(se)) { trace_sched_stat_wait(task_of(se), - rq_of(cfs_rq)->clock - se->statistics.wait_start); + rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start); } #endif schedstat_set(se->statistics.wait_start, 0); @@ -800,7 +811,7 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) /* * We are starting a new run period: */ - se->exec_start = rq_of(cfs_rq)->clock_task; + se->exec_start = rq_clock_task(rq_of(cfs_rq)); } /************************************************** @@ -809,11 +820,12 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) #ifdef CONFIG_NUMA_BALANCING /* - * numa task sample period in ms + * Approximate time to scan a full NUMA task in ms. The task scan period is + * calculated based on the tasks virtual memory size and + * numa_balancing_scan_size. */ -unsigned int sysctl_numa_balancing_scan_period_min = 100; -unsigned int sysctl_numa_balancing_scan_period_max = 100*50; -unsigned int sysctl_numa_balancing_scan_period_reset = 100*600; +unsigned int sysctl_numa_balancing_scan_period_min = 1000; +unsigned int sysctl_numa_balancing_scan_period_max = 60000; /* Portion of address space to scan in MB */ unsigned int sysctl_numa_balancing_scan_size = 256; @@ -821,2262 +833,1427 @@ unsigned int sysctl_numa_balancing_scan_size = 256; /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */ unsigned int sysctl_numa_balancing_scan_delay = 1000; -static void task_numa_placement(struct task_struct *p) +static unsigned int task_nr_scan_windows(struct task_struct *p) { - int seq; + unsigned long rss = 0; + unsigned long nr_scan_pages; - if (!p->mm) /* for example, ksmd faulting in a user's mm */ - return; - seq = ACCESS_ONCE(p->mm->numa_scan_seq); - if (p->numa_scan_seq == seq) - return; - p->numa_scan_seq = seq; + /* + * Calculations based on RSS as non-present and empty pages are skipped + * by the PTE scanner and NUMA hinting faults should be trapped based + * on resident pages + */ + nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT); + rss = get_mm_rss(p->mm); + if (!rss) + rss = nr_scan_pages; - /* FIXME: Scheduling placement policy hints go here */ + rss = round_up(rss, nr_scan_pages); + return rss / nr_scan_pages; } -/* - * Got a PROT_NONE fault for a page on @node. - */ -void task_numa_fault(int node, int pages, bool migrated) +/* For sanitys sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */ +#define MAX_SCAN_WINDOW 2560 + +static unsigned int task_scan_min(struct task_struct *p) { - struct task_struct *p = current; + unsigned int scan_size = ACCESS_ONCE(sysctl_numa_balancing_scan_size); + unsigned int scan, floor; + unsigned int windows = 1; - if (!sched_feat_numa(NUMA)) - return; + if (scan_size < MAX_SCAN_WINDOW) + windows = MAX_SCAN_WINDOW / scan_size; + floor = 1000 / windows; - /* FIXME: Allocate task-specific structure for placement policy here */ + scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); + return max_t(unsigned int, floor, scan); +} + +static unsigned int task_scan_max(struct task_struct *p) +{ + unsigned int smin = task_scan_min(p); + unsigned int smax; + + /* Watch for min being lower than max due to floor calculations */ + smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); + return max(smin, smax); +} + +static void account_numa_enqueue(struct rq *rq, struct task_struct *p) +{ + rq->nr_numa_running += (p->numa_preferred_nid != -1); + rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); +} +static void account_numa_dequeue(struct rq *rq, struct task_struct *p) +{ + rq->nr_numa_running -= (p->numa_preferred_nid != -1); + rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); +} + +struct numa_group { + atomic_t refcount; + + spinlock_t lock; /* nr_tasks, tasks */ + int nr_tasks; + pid_t gid; + struct list_head task_list; + + struct rcu_head rcu; + nodemask_t active_nodes; + unsigned long total_faults; /* - * If pages are properly placed (did not migrate) then scan slower. - * This is reset periodically in case of phase changes + * Faults_cpu is used to decide whether memory should move + * towards the CPU. As a consequence, these stats are weighted + * more by CPU use than by memory faults. */ - if (!migrated) - p->numa_scan_period = min(sysctl_numa_balancing_scan_period_max, - p->numa_scan_period + jiffies_to_msecs(10)); + unsigned long *faults_cpu; + unsigned long faults[0]; +}; - task_numa_placement(p); +/* Shared or private faults. */ +#define NR_NUMA_HINT_FAULT_TYPES 2 + +/* Memory and CPU locality */ +#define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2) + +/* Averaged statistics, and temporary buffers. */ +#define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2) + +pid_t task_numa_group_id(struct task_struct *p) +{ + return p->numa_group ? p->numa_group->gid : 0; } -static void reset_ptenuma_scan(struct task_struct *p) +static inline int task_faults_idx(int nid, int priv) { - ACCESS_ONCE(p->mm->numa_scan_seq)++; - p->mm->numa_scan_offset = 0; + return NR_NUMA_HINT_FAULT_TYPES * nid + priv; +} + +static inline unsigned long task_faults(struct task_struct *p, int nid) +{ + if (!p->numa_faults_memory) + return 0; + + return p->numa_faults_memory[task_faults_idx(nid, 0)] + + p->numa_faults_memory[task_faults_idx(nid, 1)]; +} + +static inline unsigned long group_faults(struct task_struct *p, int nid) +{ + if (!p->numa_group) + return 0; + + return p->numa_group->faults[task_faults_idx(nid, 0)] + + p->numa_group->faults[task_faults_idx(nid, 1)]; +} + +static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) +{ + return group->faults_cpu[task_faults_idx(nid, 0)] + + group->faults_cpu[task_faults_idx(nid, 1)]; } /* - * The expensive part of numa migration is done from task_work context. - * Triggered from task_tick_numa(). + * These return the fraction of accesses done by a particular task, or + * task group, on a particular numa node. The group weight is given a + * larger multiplier, in order to group tasks together that are almost + * evenly spread out between numa nodes. */ -void task_numa_work(struct callback_head *work) +static inline unsigned long task_weight(struct task_struct *p, int nid) { - unsigned long migrate, next_scan, now = jiffies; - struct task_struct *p = current; - struct mm_struct *mm = p->mm; - struct vm_area_struct *vma; - unsigned long start, end; - long pages; + unsigned long total_faults; - WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work)); + if (!p->numa_faults_memory) + return 0; + + total_faults = p->total_numa_faults; + + if (!total_faults) + return 0; + + return 1000 * task_faults(p, nid) / total_faults; +} + +static inline unsigned long group_weight(struct task_struct *p, int nid) +{ + if (!p->numa_group || !p->numa_group->total_faults) + return 0; + + return 1000 * group_faults(p, nid) / p->numa_group->total_faults; +} + +bool should_numa_migrate_memory(struct task_struct *p, struct page * page, + int src_nid, int dst_cpu) +{ + struct numa_group *ng = p->numa_group; + int dst_nid = cpu_to_node(dst_cpu); + int last_cpupid, this_cpupid; + + this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); - work->next = work; /* protect against double add */ /* - * Who cares about NUMA placement when they're dying. + * Multi-stage node selection is used in conjunction with a periodic + * migration fault to build a temporal task<->page relation. By using + * a two-stage filter we remove short/unlikely relations. * - * NOTE: make sure not to dereference p->mm before this check, - * exit_task_work() happens _after_ exit_mm() so we could be called - * without p->mm even though we still had it when we enqueued this - * work. + * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate + * a task's usage of a particular page (n_p) per total usage of this + * page (n_t) (in a given time-span) to a probability. + * + * Our periodic faults will sample this probability and getting the + * same result twice in a row, given these samples are fully + * independent, is then given by P(n)^2, provided our sample period + * is sufficiently short compared to the usage pattern. + * + * This quadric squishes small probabilities, making it less likely we + * act on an unlikely task<->page relation. */ - if (p->flags & PF_EXITING) - return; + last_cpupid = page_cpupid_xchg_last(page, this_cpupid); + if (!cpupid_pid_unset(last_cpupid) && + cpupid_to_nid(last_cpupid) != dst_nid) + return false; - /* - * We do not care about task placement until a task runs on a node - * other than the first one used by the address space. This is - * largely because migrations are driven by what CPU the task - * is running on. If it's never scheduled on another node, it'll - * not migrate so why bother trapping the fault. - */ - if (mm->first_nid == NUMA_PTE_SCAN_INIT) - mm->first_nid = numa_node_id(); - if (mm->first_nid != NUMA_PTE_SCAN_ACTIVE) { - /* Are we running on a new node yet? */ - if (numa_node_id() == mm->first_nid && - !sched_feat_numa(NUMA_FORCE)) - return; + /* Always allow migrate on private faults */ + if (cpupid_match_pid(p, last_cpupid)) + return true; - mm->first_nid = NUMA_PTE_SCAN_ACTIVE; - } + /* A shared fault, but p->numa_group has not been set up yet. */ + if (!ng) + return true; /* - * Reset the scan period if enough time has gone by. Objective is that - * scanning will be reduced if pages are properly placed. As tasks - * can enter different phases this needs to be re-examined. Lacking - * proper tracking of reference behaviour, this blunt hammer is used. + * Do not migrate if the destination is not a node that + * is actively used by this numa group. */ - migrate = mm->numa_next_reset; - if (time_after(now, migrate)) { - p->numa_scan_period = sysctl_numa_balancing_scan_period_min; - next_scan = now + msecs_to_jiffies(sysctl_numa_balancing_scan_period_reset); - xchg(&mm->numa_next_reset, next_scan); - } + if (!node_isset(dst_nid, ng->active_nodes)) + return false; /* - * Enforce maximal scan/migration frequency.. + * Source is a node that is not actively used by this + * numa group, while the destination is. Migrate. */ - migrate = mm->numa_next_scan; - if (time_before(now, migrate)) - return; - - if (p->numa_scan_period == 0) - p->numa_scan_period = sysctl_numa_balancing_scan_period_min; - - next_scan = now + msecs_to_jiffies(p->numa_scan_period); - if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate) - return; + if (!node_isset(src_nid, ng->active_nodes)) + return true; /* - * Do not set pte_numa if the current running node is rate-limited. - * This loses statistics on the fault but if we are unwilling to - * migrate to this node, it is less likely we can do useful work + * Both source and destination are nodes in active + * use by this numa group. Maximize memory bandwidth + * by migrating from more heavily used groups, to less + * heavily used ones, spreading the load around. + * Use a 1/4 hysteresis to avoid spurious page movement. */ - if (migrate_ratelimited(numa_node_id())) - return; + return group_faults(p, dst_nid) < (group_faults(p, src_nid) * 3 / 4); +} - start = mm->numa_scan_offset; - pages = sysctl_numa_balancing_scan_size; - pages <<= 20 - PAGE_SHIFT; /* MB in pages */ - if (!pages) - return; +static unsigned long weighted_cpuload(const int cpu); +static unsigned long source_load(int cpu, int type); +static unsigned long target_load(int cpu, int type); +static unsigned long capacity_of(int cpu); +static long effective_load(struct task_group *tg, int cpu, long wl, long wg); - down_read(&mm->mmap_sem); - vma = find_vma(mm, start); - if (!vma) { - reset_ptenuma_scan(p); - start = 0; - vma = mm->mmap; - } - for (; vma; vma = vma->vm_next) { - if (!vma_migratable(vma)) - continue; +/* Cached statistics for all CPUs within a node */ +struct numa_stats { + unsigned long nr_running; + unsigned long load; - /* Skip small VMAs. They are not likely to be of relevance */ - if (vma->vm_end - vma->vm_start < HPAGE_SIZE) - continue; + /* Total compute capacity of CPUs on a node */ + unsigned long compute_capacity; - /* - * Skip inaccessible VMAs to avoid any confusion between - * PROT_NONE and NUMA hinting ptes - */ - if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) - continue; + /* Approximate capacity in terms of runnable tasks on a node */ + unsigned long task_capacity; + int has_free_capacity; +}; - do { - start = max(start, vma->vm_start); - end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); - end = min(end, vma->vm_end); - pages -= change_prot_numa(vma, start, end); +/* + * XXX borrowed from update_sg_lb_stats + */ +static void update_numa_stats(struct numa_stats *ns, int nid) +{ + int smt, cpu, cpus = 0; + unsigned long capacity; - start = end; - if (pages <= 0) - goto out; - } while (end != vma->vm_end); + memset(ns, 0, sizeof(*ns)); + for_each_cpu(cpu, cpumask_of_node(nid)) { + struct rq *rq = cpu_rq(cpu); + + ns->nr_running += rq->nr_running; + ns->load += weighted_cpuload(cpu); + ns->compute_capacity += capacity_of(cpu); + + cpus++; } -out: /* - * It is possible to reach the end of the VMA list but the last few VMAs are - * not guaranteed to the vma_migratable. If they are not, we would find the - * !migratable VMA on the next scan but not reset the scanner to the start - * so check it now. + * If we raced with hotplug and there are no CPUs left in our mask + * the @ns structure is NULL'ed and task_numa_compare() will + * not find this node attractive. + * + * We'll either bail at !has_free_capacity, or we'll detect a huge + * imbalance and bail there. */ - if (vma) - mm->numa_scan_offset = start; - else - reset_ptenuma_scan(p); - up_read(&mm->mmap_sem); -} - -/* - * Drive the periodic memory faults.. - */ -void task_tick_numa(struct rq *rq, struct task_struct *curr) -{ - struct callback_head *work = &curr->numa_work; - u64 period, now; - - /* - * We don't care about NUMA placement if we don't have memory. - */ - if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work) - return; - - /* - * Using runtime rather than walltime has the dual advantage that - * we (mostly) drive the selection from busy threads and that the - * task needs to have done some actual work before we bother with - * NUMA placement. - */ - now = curr->se.sum_exec_runtime; - period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; - - if (now - curr->node_stamp > period) { - if (!curr->node_stamp) - curr->numa_scan_period = sysctl_numa_balancing_scan_period_min; - curr->node_stamp = now; - - if (!time_before(jiffies, curr->mm->numa_next_scan)) { - init_task_work(work, task_numa_work); /* TODO: move this into sched_fork() */ - task_work_add(curr, work, true); - } - } -} -#else -static void task_tick_numa(struct rq *rq, struct task_struct *curr) -{ -} -#endif /* CONFIG_NUMA_BALANCING */ - -static void -account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) -{ - update_load_add(&cfs_rq->load, se->load.weight); - if (!parent_entity(se)) - update_load_add(&rq_of(cfs_rq)->load, se->load.weight); -#ifdef CONFIG_SMP - if (entity_is_task(se)) - list_add(&se->group_node, &rq_of(cfs_rq)->cfs_tasks); -#endif - cfs_rq->nr_running++; -} - -static void -account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) -{ - update_load_sub(&cfs_rq->load, se->load.weight); - if (!parent_entity(se)) - update_load_sub(&rq_of(cfs_rq)->load, se->load.weight); - if (entity_is_task(se)) - list_del_init(&se->group_node); - cfs_rq->nr_running--; -} - -#ifdef CONFIG_FAIR_GROUP_SCHED -# ifdef CONFIG_SMP -static inline long calc_tg_weight(struct task_group *tg, struct cfs_rq *cfs_rq) -{ - long tg_weight; - - /* - * Use this CPU's actual weight instead of the last load_contribution - * to gain a more accurate current total weight. See - * update_cfs_rq_load_contribution(). - */ - tg_weight = atomic64_read(&tg->load_avg); - tg_weight -= cfs_rq->tg_load_contrib; - tg_weight += cfs_rq->load.weight; - - return tg_weight; -} - -static long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg) -{ - long tg_weight, load, shares; - - tg_weight = calc_tg_weight(tg, cfs_rq); - load = cfs_rq->load.weight; - - shares = (tg->shares * load); - if (tg_weight) - shares /= tg_weight; - - if (shares < MIN_SHARES) - shares = MIN_SHARES; - if (shares > tg->shares) - shares = tg->shares; - - return shares; -} -# else /* CONFIG_SMP */ -static inline long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg) -{ - return tg->shares; -} -# endif /* CONFIG_SMP */ -static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, - unsigned long weight) -{ - if (se->on_rq) { - /* commit outstanding execution time */ - if (cfs_rq->curr == se) - update_curr(cfs_rq); - account_entity_dequeue(cfs_rq, se); - } - - update_load_set(&se->load, weight); - - if (se->on_rq) - account_entity_enqueue(cfs_rq, se); -} - -static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); - -static void update_cfs_shares(struct cfs_rq *cfs_rq) -{ - struct task_group *tg; - struct sched_entity *se; - long shares; - - tg = cfs_rq->tg; - se = tg->se[cpu_of(rq_of(cfs_rq))]; - if (!se || throttled_hierarchy(cfs_rq)) - return; -#ifndef CONFIG_SMP - if (likely(se->load.weight == tg->shares)) - return; -#endif - shares = calc_cfs_shares(cfs_rq, tg); - - reweight_entity(cfs_rq_of(se), se, shares); -} -#else /* CONFIG_FAIR_GROUP_SCHED */ -static inline void update_cfs_shares(struct cfs_rq *cfs_rq) -{ -} -#endif /* CONFIG_FAIR_GROUP_SCHED */ - -#ifdef CONFIG_SMP - -u32 sched_get_wake_up_idle(struct task_struct *p) -{ - u32 enabled = p->flags & PF_WAKE_UP_IDLE; - - return !!enabled; -} - -int sched_set_wake_up_idle(struct task_struct *p, int wake_up_idle) -{ - int enable = !!wake_up_idle; - - if (enable) - p->flags |= PF_WAKE_UP_IDLE; - else - p->flags &= ~PF_WAKE_UP_IDLE; - - return 0; -} - -#endif /* CONFIG_SMP */ - -/* Only depends on SMP, FAIR_GROUP_SCHED may be removed when useful in lb */ -#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED) -/* - * We choose a half-life close to 1 scheduling period. - * Note: The tables below are dependent on this value. - */ -#define LOAD_AVG_PERIOD 32 -#define LOAD_AVG_MAX 47742 /* maximum possible load avg */ -#define LOAD_AVG_MAX_N 345 /* number of full periods to produce LOAD_MAX_AVG */ - -/* Precomputed fixed inverse multiplies for multiplication by y^n */ -static const u32 runnable_avg_yN_inv[] = { - 0xffffffff, 0xfa83b2da, 0xf5257d14, 0xefe4b99a, 0xeac0c6e6, 0xe5b906e6, - 0xe0ccdeeb, 0xdbfbb796, 0xd744fcc9, 0xd2a81d91, 0xce248c14, 0xc9b9bd85, - 0xc5672a10, 0xc12c4cc9, 0xbd08a39e, 0xb8fbaf46, 0xb504f333, 0xb123f581, - 0xad583ee9, 0xa9a15ab4, 0xa5fed6a9, 0xa2704302, 0x9ef5325f, 0x9b8d39b9, - 0x9837f050, 0x94f4efa8, 0x91c3d373, 0x8ea4398a, 0x8b95c1e3, 0x88980e80, - 0x85aac367, 0x82cd8698, -}; - -/* - * Precomputed \Sum y^k { 1<=k<=n }. These are floor(true_value) to prevent - * over-estimates when re-combining. - */ -static const u32 runnable_avg_yN_sum[] = { - 0, 1002, 1982, 2941, 3880, 4798, 5697, 6576, 7437, 8279, 9103, - 9909,10698,11470,12226,12966,13690,14398,15091,15769,16433,17082, - 17718,18340,18949,19545,20128,20698,21256,21802,22336,22859,23371, -}; - -/* - * Approximate: - * val * y^n, where y^32 ~= 0.5 (~1 scheduling period) - */ -static __always_inline u64 decay_load(u64 val, u64 n) -{ - unsigned int local_n; - - if (!n) - return val; - else if (unlikely(n > LOAD_AVG_PERIOD * 63)) - return 0; - - /* after bounds checking we can collapse to 32-bit */ - local_n = n; - - /* - * As y^PERIOD = 1/2, we can combine - * y^n = 1/2^(n/PERIOD) * k^(n%PERIOD) - * With a look-up table which covers k^n (n= LOAD_AVG_PERIOD)) { - val >>= local_n / LOAD_AVG_PERIOD; - local_n %= LOAD_AVG_PERIOD; - } - - val *= runnable_avg_yN_inv[local_n]; - /* We don't use SRR here since we always want to round down. */ - return val >> 32; -} - -/* - * For updates fully spanning n periods, the contribution to runnable - * average will be: \Sum 1024*y^n - * - * We can compute this reasonably efficiently by combining: - * y^PERIOD = 1/2 with precomputed \Sum 1024*y^n {for n = LOAD_AVG_MAX_N)) - return LOAD_AVG_MAX; - - /* Compute \Sum k^n combining precomputed values for k^i, \Sum k^j */ - do { - contrib /= 2; /* y^LOAD_AVG_PERIOD = 1/2 */ - contrib += runnable_avg_yN_sum[LOAD_AVG_PERIOD]; - - n -= LOAD_AVG_PERIOD; - } while (n > LOAD_AVG_PERIOD); - - contrib = decay_load(contrib, n); - return contrib + runnable_avg_yN_sum[n]; -} - -static void add_to_scaled_stat(int cpu, struct sched_avg *sa, u64 delta); -static inline void decay_scaled_stat(struct sched_avg *sa, u64 periods); - -#ifdef CONFIG_SCHED_HMP - -/* Initial task load. Newly created tasks are assigned this load. */ -unsigned int __read_mostly sched_init_task_load_pelt; -unsigned int __read_mostly sched_init_task_load_windows; -unsigned int __read_mostly sysctl_sched_init_task_load_pct = 15; - -/* - * Keep these two below in sync. One is in unit of ns and the - * other in unit of us. - */ -unsigned int __read_mostly sysctl_sched_min_runtime = 0; /* 0 ms */ -u64 __read_mostly sched_min_runtime = 0; /* 0 ms */ - -static inline unsigned int task_load(struct task_struct *p) -{ - if (sched_use_pelt) - return p->se.avg.runnable_avg_sum_scaled; - - return p->ravg.demand; -} - -unsigned int max_task_load(void) -{ - if (sched_use_pelt) - return LOAD_AVG_MAX; - - return sched_ravg_window; -} - -/* Use this knob to turn on or off HMP-aware task placement logic */ -unsigned int __read_mostly sched_enable_hmp = 0; - -/* A cpu can no longer accomodate more tasks if: - * - * rq->nr_running > sysctl_sched_spill_nr_run || - * rq->hmp_stats.cumulative_runnable_avg > sched_spill_load - */ -unsigned int __read_mostly sysctl_sched_spill_nr_run = 10; - -/* - * Control whether or not individual CPU power consumption is used to - * guide task placement. - * This sysctl can be set to a default value using boot command line arguments. - */ -unsigned int __read_mostly sysctl_sched_enable_power_aware = 0; - -/* - * This specifies the maximum percent power difference between 2 - * CPUs for them to be considered identical in terms of their - * power characteristics (i.e. they are in the same power band). - */ -unsigned int __read_mostly sysctl_sched_powerband_limit_pct = 20; - -/* - * CPUs with load greater than the sched_spill_load_threshold are not - * eligible for task placement. When all CPUs in a cluster achieve a - * load higher than this level, tasks becomes eligible for inter - * cluster migration. - */ -unsigned int __read_mostly sched_spill_load; -unsigned int __read_mostly sysctl_sched_spill_load_pct = 100; - -/* - * Tasks whose bandwidth consumption on a cpu is less than - * sched_small_task are considered as small tasks. - */ -unsigned int __read_mostly sched_small_task; -unsigned int __read_mostly sysctl_sched_small_task_pct = 10; - -/* - * Tasks with demand >= sched_heavy_task will have their - * window-based demand added to the previous window's CPU - * time when they wake up, if they have slept for at least - * one full window. This feature is disabled when the tunable - * is set to 0 (the default). - */ -#ifdef CONFIG_SCHED_FREQ_INPUT -unsigned int __read_mostly sysctl_sched_heavy_task_pct; -unsigned int __read_mostly sched_heavy_task; -#endif - -/* - * Tasks whose bandwidth consumption on a cpu is more than - * sched_upmigrate are considered "big" tasks. Big tasks will be - * considered for "up" migration, i.e migrating to a cpu with better - * capacity. - */ -unsigned int __read_mostly sched_upmigrate; -unsigned int __read_mostly sysctl_sched_upmigrate_pct = 80; - -/* - * Big tasks, once migrated, will need to drop their bandwidth - * consumption to less than sched_downmigrate before they are "down" - * migrated. - */ -unsigned int __read_mostly sched_downmigrate; -unsigned int __read_mostly sysctl_sched_downmigrate_pct = 60; - -/* - * Tasks whose nice value is > sysctl_sched_upmigrate_min_nice are never - * considered as "big" tasks. - */ -static int __read_mostly sched_upmigrate_min_nice = 15; -int __read_mostly sysctl_sched_upmigrate_min_nice = 15; - -/* - * Scheduler boost is a mechanism to temporarily place tasks on CPUs - * with higher capacity than those where a task would have normally - * ended up with their load characteristics. Any entity enabling - * boost is responsible for disabling it as well. - */ -unsigned int sysctl_sched_boost; - -static inline int available_cpu_capacity(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - return rq->capacity; -} - -void set_hmp_defaults(void) -{ - sched_spill_load = - pct_to_real(sysctl_sched_spill_load_pct); - - sched_small_task = - pct_to_real(sysctl_sched_small_task_pct); - - sched_upmigrate = - pct_to_real(sysctl_sched_upmigrate_pct); - - sched_downmigrate = - pct_to_real(sysctl_sched_downmigrate_pct); - -#ifdef CONFIG_SCHED_FREQ_INPUT - sched_heavy_task = - pct_to_real(sysctl_sched_heavy_task_pct); -#endif - - sched_init_task_load_pelt = - div64_u64((u64)sysctl_sched_init_task_load_pct * - (u64)LOAD_AVG_MAX, 100); - - sched_init_task_load_windows = - div64_u64((u64)sysctl_sched_init_task_load_pct * - (u64)sched_ravg_window, 100); - - sched_upmigrate_min_nice = sysctl_sched_upmigrate_min_nice; -} - -u32 sched_get_init_task_load(struct task_struct *p) -{ - return p->init_load_pct; -} - -int sched_set_init_task_load(struct task_struct *p, int init_load_pct) -{ - if (init_load_pct < 0 || init_load_pct > 100) - return -EINVAL; - - p->init_load_pct = init_load_pct; - - return 0; -} - -int sched_set_cpu_prefer_idle(int cpu, int prefer_idle) -{ - struct rq *rq = cpu_rq(cpu); - - rq->prefer_idle = !!prefer_idle; - - return 0; -} - -int sched_get_cpu_prefer_idle(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - return rq->prefer_idle; -} - -int sched_set_cpu_mostly_idle_load(int cpu, int mostly_idle_pct) -{ - struct rq *rq = cpu_rq(cpu); - - if (mostly_idle_pct < 0 || mostly_idle_pct > 100) - return -EINVAL; - - rq->mostly_idle_load = pct_to_real(mostly_idle_pct); - - return 0; -} - -int sched_set_cpu_mostly_idle_freq(int cpu, unsigned int mostly_idle_freq) -{ - struct rq *rq = cpu_rq(cpu); - - if (mostly_idle_freq > rq->max_possible_freq) - return -EINVAL; - - rq->mostly_idle_freq = mostly_idle_freq; - - return 0; -} - -unsigned int sched_get_cpu_mostly_idle_freq(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - return rq->mostly_idle_freq; -} - -int sched_get_cpu_mostly_idle_load(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - int mostly_idle_pct; - - mostly_idle_pct = real_to_pct(rq->mostly_idle_load); - - return mostly_idle_pct; -} - -int sched_set_cpu_mostly_idle_nr_run(int cpu, int nr_run) -{ - struct rq *rq = cpu_rq(cpu); - - rq->mostly_idle_nr_run = nr_run; - - return 0; -} - -int sched_get_cpu_mostly_idle_nr_run(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - return rq->mostly_idle_nr_run; -} - -#ifdef CONFIG_CGROUP_SCHED - -static inline int upmigrate_discouraged(struct task_struct *p) -{ - return task_group(p)->upmigrate_discouraged; -} - -#else - -static inline int upmigrate_discouraged(struct task_struct *p) -{ - return 0; -} - -#endif - -/* Is a task "big" on its current cpu */ -static inline int is_big_task(struct task_struct *p) -{ - u64 load = task_load(p); - int nice = TASK_NICE(p); - - if (nice > sched_upmigrate_min_nice || upmigrate_discouraged(p)) - return 0; - - load = scale_load_to_cpu(load, task_cpu(p)); - - return load > sched_upmigrate; -} - -/* Is a task "small" on the minimum capacity CPU */ -static inline int is_small_task(struct task_struct *p) -{ - u64 load = task_load(p); - load *= (u64)max_load_scale_factor; - load /= 1024; - return load < sched_small_task; -} - -static inline u64 cpu_load(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - return scale_load_to_cpu(rq->hmp_stats.cumulative_runnable_avg, cpu); -} - -static inline u64 cpu_load_sync(int cpu, int sync) -{ - struct rq *rq = cpu_rq(cpu); - u64 load; - - load = rq->hmp_stats.cumulative_runnable_avg; - - /* - * If load is being checked in a sync wakeup environment, - * we may want to discount the load of the currently running - * task. - */ - if (sync && cpu == smp_processor_id()) { - if (load > rq->curr->ravg.demand) - load -= rq->curr->ravg.demand; - else - load = 0; - } - - return scale_load_to_cpu(load, cpu); -} - -static int -spill_threshold_crossed(u64 task_load, u64 cpu_load, struct rq *rq) -{ - u64 total_load = task_load + cpu_load; - - if (total_load > sched_spill_load || - (rq->nr_running + 1) > sysctl_sched_spill_nr_run) - return 1; - - return 0; -} - -int mostly_idle_cpu(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - return cpu_load(cpu) <= rq->mostly_idle_load - && rq->nr_running <= rq->mostly_idle_nr_run - && !sched_cpu_high_irqload(cpu); -} - -static int mostly_idle_cpu_sync(int cpu, u64 load, int sync) -{ - struct rq *rq = cpu_rq(cpu); - int nr_running; - - nr_running = rq->nr_running; - - /* - * Sync wakeups mean that the waker task will go to sleep - * soon so we should discount its load from this test. - */ - if (sync && cpu == smp_processor_id()) - nr_running--; - - return load <= rq->mostly_idle_load && - nr_running <= rq->mostly_idle_nr_run; -} - -static int boost_refcount; -static DEFINE_SPINLOCK(boost_lock); -static DEFINE_MUTEX(boost_mutex); - -static void boost_kick_cpus(void) -{ - int i; - - for_each_online_cpu(i) { - if (cpu_rq(i)->capacity != max_capacity) - boost_kick(i); - } -} - -int sched_boost(void) -{ - return boost_refcount > 0; -} - -int sched_set_boost(int enable) -{ - unsigned long flags; - int ret = 0; - int old_refcount; - - if (!sched_enable_hmp) - return -EINVAL; - - spin_lock_irqsave(&boost_lock, flags); - - old_refcount = boost_refcount; - - if (enable == 1) { - boost_refcount++; - } else if (!enable) { - if (boost_refcount >= 1) - boost_refcount--; - else - ret = -EINVAL; - } else { - ret = -EINVAL; - } - - if (!old_refcount && boost_refcount) - boost_kick_cpus(); - - trace_sched_set_boost(boost_refcount); - spin_unlock_irqrestore(&boost_lock, flags); - - return ret; -} - -int sched_boost_handler(struct ctl_table *table, int write, - void __user *buffer, size_t *lenp, - loff_t *ppos) -{ - int ret; - - mutex_lock(&boost_mutex); - if (!write) - sysctl_sched_boost = sched_boost(); - - ret = proc_dointvec(table, write, buffer, lenp, ppos); - if (ret || !write) - goto done; - - ret = (sysctl_sched_boost <= 1) ? - sched_set_boost(sysctl_sched_boost) : -EINVAL; - -done: - mutex_unlock(&boost_mutex); - return ret; -} - -static inline int is_cpu_throttling_imminent(int cpu); -#ifdef VENDOR_EDIT -void down_migrate_task(struct task_struct *p) -{ - if (p->ravg.mitigated) + if (!cpus) return; - if (sysctl_thermal_aware_scheduling) - p->ravg.mitigated = 1; -} -#endif - -/* - * Task will fit on a cpu if it's bandwidth consumption on that cpu - * will be less than sched_upmigrate. A big task that was previously - * "up" migrated will be considered fitting on "little" cpu if its - * bandwidth consumption on "little" cpu will be less than - * sched_downmigrate. This will help avoid frequenty migrations for - * tasks with load close to the upmigrate threshold - */ - -static int task_load_will_fit(struct task_struct *p, u64 tload, int cpu) -{ - struct rq *prev_rq = cpu_rq(task_cpu(p)); - struct rq *rq = cpu_rq(cpu); - int upmigrate, nice; -#ifdef VENDOR_EDIT - int thermal_mitigation = 0; - int prev_cpu_throttled = 0, cpu_throttled = 0, ta_enabled = 0; - int prev_cpu = task_cpu(p); -#endif - - if (rq->capacity == max_capacity) - return 1; - -#ifdef VENDOR_EDIT - if (p->ravg.mitigated) - return 1; -#endif - if (sched_boost()) { - if (rq->capacity > prev_rq->capacity) - return 1; - } else { - nice = TASK_NICE(p); - if (nice > sched_upmigrate_min_nice || upmigrate_discouraged(p)) - return 1; - - upmigrate = sched_upmigrate; -#ifdef VENDOR_EDIT - if (sysctl_thermal_aware_scheduling == SCHED_TA_THERMAL_ONLY) { - prev_cpu_throttled = is_cpu_throttling_imminent(prev_cpu); - cpu_throttled = is_cpu_throttling_imminent(cpu); - if (prev_cpu_throttled || cpu_throttled) - ta_enabled = 1; - } else if (sysctl_thermal_aware_scheduling == SCHED_TA_ALWAYS_ON) - ta_enabled = 1; - - if (ta_enabled && cpu != prev_cpu - && prev_rq->max_possible_capacity > rq->max_possible_capacity) { - thermal_mitigation = 1; - tload = scale_load_to_cpu(task_load(p), prev_cpu); - } -#endif - -#ifdef VENDOR_EDIT - if (prev_rq->capacity > rq->capacity || thermal_mitigation) -#else - if (prev_rq->capacity > rq->capacity) -#endif - upmigrate = sched_downmigrate; - - if (tload < upmigrate) { -#ifdef VENDOR_EDIT - if (thermal_mitigation) - down_migrate_task(p); -#endif - return 1; - } - } - - return 0; -} + /* smt := ceil(cpus / capacity), assumes: 1 < smt_power < 2 */ + smt = DIV_ROUND_UP(SCHED_CAPACITY_SCALE * cpus, ns->compute_capacity); + capacity = cpus / smt; /* cores */ -static int task_will_fit(struct task_struct *p, int cpu) -{ - u64 tload = scale_load_to_cpu(task_load(p), cpu); - return task_load_will_fit(p, tload, cpu); + ns->task_capacity = min_t(unsigned, capacity, + DIV_ROUND_CLOSEST(ns->compute_capacity, SCHED_CAPACITY_SCALE)); + ns->has_free_capacity = (ns->nr_running < ns->task_capacity); } -static int eligible_cpu(u64 task_load, u64 cpu_load, int cpu, int sync, unsigned task_mitigated) -{ - struct rq *rq = cpu_rq(cpu); - - if (sched_cpu_high_irqload(cpu)) - return 0; +struct task_numa_env { + struct task_struct *p; - if (mostly_idle_cpu_sync(cpu, cpu_load, sync)) - return 1; + int src_cpu, src_nid; + int dst_cpu, dst_nid; - if (rq->max_possible_capacity != max_possible_capacity) { -#ifdef VENDOR_EDIT - /* Current demand is still high, bail all small cpus out. - * select_best_cpu() will choose proper cpu to run */ - if (task_mitigated) - return 1; -#endif - return !spill_threshold_crossed(task_load, cpu_load, rq); - } + struct numa_stats src_stats, dst_stats; - return 0; -} + int imbalance_pct; -struct cpu_pwr_stats __weak *get_cpu_pwr_stats(void) -{ - return NULL; -} + struct task_struct *best_task; + long best_imp; + int best_cpu; +}; -int power_delta_exceeded(unsigned int cpu_cost, unsigned int base_cost) +static void task_numa_assign(struct task_numa_env *env, + struct task_struct *p, long imp) { - int delta, cost_limit; - - if (!base_cost || cpu_cost == base_cost) - return 0; + if (env->best_task) + put_task_struct(env->best_task); + if (p) + get_task_struct(p); - delta = cpu_cost - base_cost; - cost_limit = div64_u64((u64)sysctl_sched_powerband_limit_pct * - (u64)base_cost, 100); - return abs(delta) > cost_limit; + env->best_task = p; + env->best_imp = imp; + env->best_cpu = env->dst_cpu; } -unsigned int power_cost_at_freq(int cpu, unsigned int freq) +static bool load_too_imbalanced(long src_load, long dst_load, + struct task_numa_env *env) { - int i = 0; - struct cpu_pwr_stats *per_cpu_info = get_cpu_pwr_stats(); - struct cpu_pstate_pwr *costs; - - if (!per_cpu_info || !per_cpu_info[cpu].ptable || - !sysctl_sched_enable_power_aware) - /* When power aware scheduling is not in use, or CPU - * power data is not available, just use the CPU - * capacity as a rough stand-in for real CPU power - * numbers, assuming bigger CPUs are more power - * hungry. */ - return cpu_rq(cpu)->max_possible_capacity; - - if (!freq) - freq = min_max_freq; - - costs = per_cpu_info[cpu].ptable; + long imb, old_imb; + long orig_src_load, orig_dst_load; + long src_capacity, dst_capacity; - while (costs[i].freq != 0) { - if (costs[i].freq >= freq || - costs[i+1].freq == 0) - return costs[i].power; - i++; - } - BUG(); -} + /* + * The load is corrected for the CPU capacity available on each node. + * + * src_load dst_load + * ------------ vs --------- + * src_capacity dst_capacity + */ + src_capacity = env->src_stats.compute_capacity; + dst_capacity = env->dst_stats.compute_capacity; -/* Return the cost of running task p on CPU cpu. This function - * currently assumes that task p is the only task which will run on - * the CPU. */ -static unsigned int power_cost(u64 task_load, int cpu) -{ - unsigned int task_freq, cur_freq; - struct rq *rq = cpu_rq(cpu); - u64 demand; + /* We care about the slope of the imbalance, not the direction. */ + if (dst_load < src_load) + swap(dst_load, src_load); - if (!sysctl_sched_enable_power_aware) - return rq->max_possible_capacity; + /* Is the difference below the threshold? */ + imb = dst_load * src_capacity * 100 - + src_load * dst_capacity * env->imbalance_pct; + if (imb <= 0) + return false; - /* calculate % of max freq needed */ - demand = task_load * 100; - demand = div64_u64(demand, max_task_load()); + /* + * The imbalance is above the allowed threshold. + * Compare it with the old imbalance. + */ + orig_src_load = env->src_stats.load; + orig_dst_load = env->dst_stats.load; - task_freq = demand * rq->max_possible_freq; - task_freq /= 100; /* khz needed */ + if (orig_dst_load < orig_src_load) + swap(orig_dst_load, orig_src_load); - cur_freq = rq->cur_freq; - task_freq = max(cur_freq, task_freq); + old_imb = orig_dst_load * src_capacity * 100 - + orig_src_load * dst_capacity * env->imbalance_pct; - return power_cost_at_freq(cpu, task_freq); + /* Would this change make things worse? */ + return (imb > old_imb); } -static int best_small_task_cpu(struct task_struct *p, int sync) -{ - int best_busy_cpu = -1, fallback_cpu = -1; - int min_cstate_cpu = -1; - int min_cstate = INT_MAX; - int cpu_cost, min_cost = INT_MAX; - int i = task_cpu(p), prev_cpu; - int hmp_capable; - u64 tload, cpu_load, min_load = ULLONG_MAX; - cpumask_t temp; - cpumask_t search_cpu; - cpumask_t fb_search_cpu = CPU_MASK_NONE; - struct rq *rq; - - cpumask_and(&temp, &mpc_mask, cpu_possible_mask); - hmp_capable = !cpumask_full(&temp); - - cpumask_and(&search_cpu, tsk_cpus_allowed(p), cpu_online_mask); - if (unlikely(cpumask_empty(&search_cpu))) - return task_cpu(p); - if (unlikely(!cpumask_test_cpu(i, &search_cpu))) - i = cpumask_first(&search_cpu); +/* + * This checks if the overall compute and NUMA accesses of the system would + * be improved if the source tasks was migrated to the target dst_cpu taking + * into account that it might be best if task running on the dst_cpu should + * be exchanged with the source task + */ +static void task_numa_compare(struct task_numa_env *env, + long taskimp, long groupimp) +{ + struct rq *src_rq = cpu_rq(env->src_cpu); + struct rq *dst_rq = cpu_rq(env->dst_cpu); + struct task_struct *cur; + long src_load, dst_load; + long load; + long imp = env->p->numa_group ? groupimp : taskimp; + long moveimp = imp; - do { - rq = cpu_rq(i); + rcu_read_lock(); - trace_sched_cpu_load(rq, idle_cpu(i), - mostly_idle_cpu_sync(i, - cpu_load_sync(i, sync), sync), - sched_irqload(i), - power_cost(scale_load_to_cpu(task_load(p), - i), i), - cpu_temp(i)); - - if (rq->max_possible_capacity == max_possible_capacity && - hmp_capable) { - cpumask_and(&fb_search_cpu, &search_cpu, - &rq->freq_domain_cpumask); - cpumask_andnot(&search_cpu, &search_cpu, - &rq->freq_domain_cpumask); - continue; - } + raw_spin_lock_irq(&dst_rq->lock); + cur = dst_rq->curr; + /* + * No need to move the exiting task, and this ensures that ->curr + * wasn't reaped and thus get_task_struct() in task_numa_assign() + * is safe under RCU read lock. + * Note that rcu_read_lock() itself can't protect from the final + * put_task_struct() after the last schedule(). + */ + if ((cur->flags & PF_EXITING) || is_idle_task(cur)) + cur = NULL; + raw_spin_unlock_irq(&dst_rq->lock); - cpumask_clear_cpu(i, &search_cpu); + /* + * Because we have preemption enabled we can get migrated around and + * end try selecting ourselves (current == env->p) as a swap candidate. + */ + if (cur == env->p) + goto unlock; - if (sched_cpu_high_irqload(i)) - continue; + /* + * "imp" is the fault differential for the source task between the + * source and destination node. Calculate the total differential for + * the source task and potential destination task. The more negative + * the value is, the more rmeote accesses that would be expected to + * be incurred if the tasks were swapped. + */ + if (cur) { + /* Skip this swap candidate if cannot move to the source cpu */ + if (!cpumask_test_cpu(env->src_cpu, tsk_cpus_allowed(cur))) + goto unlock; - if (idle_cpu(i) && rq->cstate) { - if (rq->cstate < min_cstate) { - min_cstate_cpu = i; - min_cstate = rq->cstate; - } - continue; + /* + * If dst and source tasks are in the same NUMA group, or not + * in any group then look only at task weights. + */ + if (cur->numa_group == env->p->numa_group) { + imp = taskimp + task_weight(cur, env->src_nid) - + task_weight(cur, env->dst_nid); + /* + * Add some hysteresis to prevent swapping the + * tasks within a group over tiny differences. + */ + if (cur->numa_group) + imp -= imp/16; + } else { + /* + * Compare the group weights. If a task is all by + * itself (not part of a group), use the task weight + * instead. + */ + if (cur->numa_group) + imp += group_weight(cur, env->src_nid) - + group_weight(cur, env->dst_nid); + else + imp += task_weight(cur, env->src_nid) - + task_weight(cur, env->dst_nid); } + } - cpu_load = cpu_load_sync(i, sync); - if (mostly_idle_cpu_sync(i, cpu_load, sync)) - return i; - } while ((i = cpumask_first(&search_cpu)) < nr_cpu_ids); - - if (min_cstate_cpu != -1) - return min_cstate_cpu; - - cpumask_and(&search_cpu, tsk_cpus_allowed(p), cpu_online_mask); - cpumask_andnot(&search_cpu, &search_cpu, &fb_search_cpu); - for_each_cpu(i, &search_cpu) { - rq = cpu_rq(i); - prev_cpu = (i == task_cpu(p)); + if (imp <= env->best_imp && moveimp <= env->best_imp) + goto unlock; - if (sched_cpu_high_irqload(i)) - continue; + if (!cur) { + /* Is there capacity at our destination? */ + if (env->src_stats.nr_running <= env->src_stats.task_capacity && + !env->dst_stats.has_free_capacity) + goto unlock; - tload = scale_load_to_cpu(task_load(p), i); - cpu_load = cpu_load_sync(i, sync); - if (!spill_threshold_crossed(tload, cpu_load, rq)) { - if (cpu_load < min_load || - (prev_cpu && cpu_load == min_load)) { - min_load = cpu_load; - best_busy_cpu = i; - } - } + goto balance; } - if (best_busy_cpu != -1) - return best_busy_cpu; + /* Balance doesn't matter much if we're running a task per cpu */ + if (imp > env->best_imp && src_rq->nr_running == 1 && + dst_rq->nr_running == 1) + goto assign; - for_each_cpu(i, &fb_search_cpu) { - rq = cpu_rq(i); - prev_cpu = (i == task_cpu(p)); - - tload = scale_load_to_cpu(task_load(p), i); - cpu_cost = power_cost(tload, i); - if (cpu_cost < min_cost || - (prev_cpu && cpu_cost == min_cost)) { - fallback_cpu = i; - min_cost = cpu_cost; + /* + * In the overloaded case, try and keep the load balanced. + */ +balance: + load = task_h_load(env->p); + dst_load = env->dst_stats.load + load; + src_load = env->src_stats.load - load; + + if (moveimp > imp && moveimp > env->best_imp) { + /* + * If the improvement from just moving env->p direction is + * better than swapping tasks around, check if a move is + * possible. Store a slightly smaller score than moveimp, + * so an actually idle CPU will win. + */ + if (!load_too_imbalanced(src_load, dst_load, env)) { + imp = moveimp - 1; + cur = NULL; + goto assign; } } - return fallback_cpu; -} - -#define UP_MIGRATION 1 -#define DOWN_MIGRATION 2 -#define EA_MIGRATION 3 -#define IRQLOAD_MIGRATION 4 - -static int skip_freq_domain(struct rq *task_rq, struct rq *rq, int reason) -{ - int skip; - - if (!reason) - return 0; - - switch (reason) { - case UP_MIGRATION: - skip = rq->capacity <= task_rq->capacity; - break; - - case DOWN_MIGRATION: - skip = rq->capacity >= task_rq->capacity; - break; - - case EA_MIGRATION: -#ifdef VENDOR_EDIT - if (!sysctl_thermal_aware_scheduling) -#endif - skip = rq->capacity != task_rq->capacity; - break; - - case IRQLOAD_MIGRATION: - /* Purposely fall through */ + if (imp <= env->best_imp) + goto unlock; - default: - return 0; + if (cur) { + load = task_h_load(cur); + dst_load -= load; + src_load += load; } - return skip; -} - -static int skip_cpu(struct rq *task_rq, struct rq *rq, int cpu, - u64 task_load, int reason) -{ - int skip; - - if (!reason) - return 0; - - if (is_reserved(cpu)) - return 1; - - switch (reason) { - case EA_MIGRATION: - skip = power_cost(task_load, cpu) > - power_cost(task_load, cpu_of(task_rq)); - break; - - case IRQLOAD_MIGRATION: - /* Purposely fall through */ + if (load_too_imbalanced(src_load, dst_load, env)) + goto unlock; - default: - skip = (rq == task_rq); - } + /* + * One idle CPU per node is evaluated for a task numa move. + * Call select_idle_sibling to maybe find a better one. + */ + if (!cur) + env->dst_cpu = select_idle_sibling(env->p, env->dst_cpu); - return skip; +assign: + task_numa_assign(env, cur, imp); +unlock: + rcu_read_unlock(); } -/* - * Select a single cpu in cluster as target for packing, iff cluster frequency - * is less than a threshold level - */ -static int select_packing_target(struct task_struct *p, int best_cpu) +static void task_numa_find_cpu(struct task_numa_env *env, + long taskimp, long groupimp) { - struct rq *rq = cpu_rq(best_cpu); - struct cpumask search_cpus; - int i; - int min_cost = INT_MAX; - int target = best_cpu; - - if (rq->cur_freq >= rq->mostly_idle_freq) - return best_cpu; - - /* Don't pack if current freq is low because of throttling */ - if (rq->max_freq <= rq->mostly_idle_freq) - return best_cpu; - - cpumask_and(&search_cpus, tsk_cpus_allowed(p), cpu_online_mask); - cpumask_and(&search_cpus, &search_cpus, &rq->freq_domain_cpumask); + int cpu; - /* Pick the first lowest power cpu as target */ - for_each_cpu(i, &search_cpus) { - int cost = power_cost(scale_load_to_cpu(task_load(p), i), i); + for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) { + /* Skip this CPU if the source task cannot migrate */ + if (!cpumask_test_cpu(cpu, tsk_cpus_allowed(env->p))) + continue; - if (cost < min_cost && !sched_cpu_high_irqload(i)) { - target = i; - min_cost = cost; - } + env->dst_cpu = cpu; + task_numa_compare(env, taskimp, groupimp); } - - return target; } -/* - * Should task be woken to any available idle cpu? - * - * Waking tasks to idle cpu has mixed implications on both performance and - * power. In many cases, scheduler can't estimate correctly impact of using idle - * cpus on either performance or power. PF_WAKE_UP_IDLE allows external kernel - * module to pass a strong hint to scheduler that the task in question should be - * woken to idle cpu, generally to improve performance. - */ -static inline int wake_to_idle(struct task_struct *p) -{ - return (current->flags & PF_WAKE_UP_IDLE) || - (p->flags & PF_WAKE_UP_IDLE); -} - -/* return cheapest cpu that can fit this task */ -static int select_best_cpu(struct task_struct *p, int target, int reason, - int sync) -{ - int i, j, prev_cpu, best_cpu = -1; - int fallback_idle_cpu = -1, min_cstate_cpu = -1; - int cpu_cost, min_cost = INT_MAX; - int min_idle_cost = INT_MAX, min_busy_cost = INT_MAX; - u64 tload, cpu_load; - u64 min_load = ULLONG_MAX, min_fallback_load = ULLONG_MAX; - int small_task = is_small_task(p); - int boost = sched_boost(); - int cstate, min_cstate = INT_MAX; - int prefer_idle = -1; - int prefer_idle_override = 0; - cpumask_t search_cpus; - struct rq *trq; - - if (reason) { - prefer_idle = 1; - prefer_idle_override = 1; - } - - if (wake_to_idle(p)) { - prefer_idle = 1; - prefer_idle_override = 1; - small_task = 0; - /* - * If wake to idle and sync are both set prefer wake to idle - * since sync is a weak hint that might not always be correct. - */ - sync = 0; - } - - if (small_task && !boost) { - best_cpu = best_small_task_cpu(p, sync); - prefer_idle = 0; /* For sched_task_load tracepoint */ - goto done; - } - - trq = task_rq(p); - cpumask_and(&search_cpus, tsk_cpus_allowed(p), cpu_online_mask); - for_each_cpu(i, &search_cpus) { - struct rq *rq = cpu_rq(i); - - trace_sched_cpu_load(cpu_rq(i), idle_cpu(i), - mostly_idle_cpu_sync(i, - cpu_load_sync(i, sync), sync), - sched_irqload(i), - power_cost(scale_load_to_cpu(task_load(p), - i), i), - cpu_temp(i)); - - if (skip_freq_domain(trq, rq, reason)) { - cpumask_andnot(&search_cpus, &search_cpus, - &rq->freq_domain_cpumask); - continue; - } - - tload = scale_load_to_cpu(task_load(p), i); - if (skip_cpu(trq, rq, i, tload, reason)) - continue; - - prev_cpu = (i == task_cpu(p)); - - /* - * The least-loaded mostly-idle CPU where the task - * won't fit is our fallback if we can't find a CPU - * where the task will fit. - */ - if (!task_load_will_fit(p, tload, i)) { - for_each_cpu_and(j, &search_cpus, - &rq->freq_domain_cpumask) { - cpu_load = cpu_load_sync(j, sync); - if (mostly_idle_cpu_sync(j, cpu_load, sync) && - !sched_cpu_high_irqload(j)) { - if (cpu_load < min_fallback_load || - (cpu_load == min_fallback_load && - j == task_cpu(p))) { - min_fallback_load = cpu_load; - fallback_idle_cpu = j; - } - } - } - cpumask_andnot(&search_cpus, &search_cpus, - &rq->freq_domain_cpumask); - continue; - } - - /* Set prefer_idle based on the cpu where task will first fit */ - if (prefer_idle == -1) - prefer_idle = cpu_rq(i)->prefer_idle; - - cpu_load = cpu_load_sync(i, sync); - if (!eligible_cpu(tload, cpu_load, i, sync, p->ravg.mitigated)) - continue; - - /* - * The task will fit on this CPU, and the CPU is either - * mostly_idle or not max capacity and can fit it under - * spill. - */ - - cpu_cost = power_cost(tload, i); - - /* - * If the task fits in a CPU in a lower power band, that - * overrides load and C-state. - */ - if (power_delta_exceeded(cpu_cost, min_cost)) { - if (cpu_cost > min_cost) - continue; - - min_cost = cpu_cost; - min_load = ULLONG_MAX; - min_cstate = INT_MAX; - min_cstate_cpu = -1; - best_cpu = -1; - if (!prefer_idle_override) - prefer_idle = cpu_rq(i)->prefer_idle; - } - - /* - * Partition CPUs based on whether they are completely idle - * or not. For completely idle CPUs we choose the one in - * the lowest C-state and then break ties with power cost. - * - * For sync wakeups we only consider the waker CPU as idle if - * prefer_idle is set. Otherwise if prefer_idle is unset sync - * wakeups will get biased away from the waker CPU. - */ - if (idle_cpu(i) || (sync && i == smp_processor_id() - && prefer_idle && cpu_rq(i)->nr_running == 1)) { - cstate = cpu_rq(i)->cstate; - - if (cstate > min_cstate) - continue; - - if (cstate < min_cstate) { - min_idle_cost = cpu_cost; - min_cstate = cstate; - min_cstate_cpu = i; - continue; - } - - if (cpu_cost < min_idle_cost || - (prev_cpu && cpu_cost == min_idle_cost)) { - min_idle_cost = cpu_cost; - min_cstate_cpu = i; - } - - continue; - } +static int task_numa_migrate(struct task_struct *p) +{ + struct task_numa_env env = { + .p = p, - /* - * For CPUs that are not completely idle, pick one with the - * lowest load and break ties with power cost - */ - if (cpu_load > min_load) - continue; + .src_cpu = task_cpu(p), + .src_nid = task_node(p), - if (cpu_load < min_load) { - min_load = cpu_load; - min_busy_cost = cpu_cost; - best_cpu = i; - continue; - } + .imbalance_pct = 112, - /* - * The load is equal to the previous selected CPU. - * This is rare but when it does happen opt for the - * more power efficient CPU option. - */ - if (cpu_cost < min_busy_cost || - (prev_cpu && cpu_cost == min_busy_cost)) { - min_busy_cost = cpu_cost; - best_cpu = i; - } - } + .best_task = NULL, + .best_imp = 0, + .best_cpu = -1 + }; + struct sched_domain *sd; + unsigned long taskweight, groupweight; + int nid, ret; + long taskimp, groupimp; /* - * Don't need to check !sched_cpu_high_irqload(best_cpu) because - * best_cpu cannot have high irq load. + * Pick the lowest SD_NUMA domain, as that would have the smallest + * imbalance and would be the first to start moving tasks about. + * + * And we want to avoid any moving of tasks about, as that would create + * random movement of tasks -- counter the numa conditions we're trying + * to satisfy here. */ - if (min_cstate_cpu >= 0 && (prefer_idle > 0 || best_cpu < 0 || - !mostly_idle_cpu_sync(best_cpu, min_load, sync))) - best_cpu = min_cstate_cpu; -done: - if (best_cpu < 0) { - if (unlikely(fallback_idle_cpu < 0)) - /* - * For the lack of a better choice just use - * prev_cpu. We may just benefit from having - * a hot cache. - */ - best_cpu = task_cpu(p); - else - best_cpu = fallback_idle_cpu; - } - - if (cpu_rq(best_cpu)->mostly_idle_freq && !prefer_idle_override) - best_cpu = select_packing_target(p, best_cpu); + rcu_read_lock(); + sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu)); + if (sd) + env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; + rcu_read_unlock(); /* - * prefer_idle is initialized towards middle of function. Leave this - * tracepoint towards end to capture prefer_idle flag used for this - * instance of wakeup. - */ - trace_sched_task_load(p, small_task, boost, reason, sync, prefer_idle); - - return best_cpu; -} - -static void -inc_nr_big_small_task(struct hmp_sched_stats *stats, struct task_struct *p) -{ - if (!sched_enable_hmp || sched_disable_window_stats) - return; + * Cpusets can break the scheduler domain tree into smaller + * balance domains, some of which do not cross NUMA boundaries. + * Tasks that are "trapped" in such domains cannot be migrated + * elsewhere, so there is no point in (re)trying. + */ + if (unlikely(!sd)) { + p->numa_preferred_nid = task_node(p); + return -EINVAL; + } - if (is_big_task(p)) - stats->nr_big_tasks++; - else if (is_small_task(p)) - stats->nr_small_tasks++; -} + taskweight = task_weight(p, env.src_nid); + groupweight = group_weight(p, env.src_nid); + update_numa_stats(&env.src_stats, env.src_nid); + env.dst_nid = p->numa_preferred_nid; + taskimp = task_weight(p, env.dst_nid) - taskweight; + groupimp = group_weight(p, env.dst_nid) - groupweight; + update_numa_stats(&env.dst_stats, env.dst_nid); -static void -dec_nr_big_small_task(struct hmp_sched_stats *stats, struct task_struct *p) -{ - if (!sched_enable_hmp || sched_disable_window_stats) - return; + /* Try to find a spot on the preferred nid. */ + task_numa_find_cpu(&env, taskimp, groupimp); - if (is_big_task(p)) - stats->nr_big_tasks--; - else if (is_small_task(p)) - stats->nr_small_tasks--; + /* No space available on the preferred nid. Look elsewhere. */ + if (env.best_cpu == -1) { + for_each_online_node(nid) { + if (nid == env.src_nid || nid == p->numa_preferred_nid) + continue; - BUG_ON(stats->nr_big_tasks < 0 || stats->nr_small_tasks < 0); -} + /* Only consider nodes where both task and groups benefit */ + taskimp = task_weight(p, nid) - taskweight; + groupimp = group_weight(p, nid) - groupweight; + if (taskimp < 0 && groupimp < 0) + continue; -static void -inc_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) -{ - inc_nr_big_small_task(&rq->hmp_stats, p); - if (change_cra) - inc_cumulative_runnable_avg(&rq->hmp_stats, p); -} + env.dst_nid = nid; + update_numa_stats(&env.dst_stats, env.dst_nid); + task_numa_find_cpu(&env, taskimp, groupimp); + } + } -static void -dec_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) -{ - dec_nr_big_small_task(&rq->hmp_stats, p); - if (change_cra) - dec_cumulative_runnable_avg(&rq->hmp_stats, p); -} + /* + * If the task is part of a workload that spans multiple NUMA nodes, + * and is migrating into one of the workload's active nodes, remember + * this node as the task's preferred numa node, so the workload can + * settle down. + * A task that migrated to a second choice node will be better off + * trying for a better one later. Do not set the preferred node here. + */ + if (p->numa_group) { + if (env.best_cpu == -1) + nid = env.src_nid; + else + nid = env.dst_nid; -static void reset_hmp_stats(struct hmp_sched_stats *stats, int reset_cra) -{ - stats->nr_big_tasks = stats->nr_small_tasks = 0; - if (reset_cra) - stats->cumulative_runnable_avg = 0; -} + if (node_isset(nid, p->numa_group->active_nodes)) + sched_setnuma(p, env.dst_nid); + } + /* No better CPU than the current one was found. */ + if (env.best_cpu == -1) + return -EAGAIN; -#ifdef CONFIG_CFS_BANDWIDTH + /* + * Reset the scan period if the task is being rescheduled on an + * alternative node to recheck if the tasks is now properly placed. + */ + p->numa_scan_period = task_scan_min(p); -static inline struct task_group *next_task_group(struct task_group *tg) -{ - tg = list_entry_rcu(tg->list.next, typeof(struct task_group), list); + if (env.best_task == NULL) { + ret = migrate_task_to(p, env.best_cpu); + if (ret != 0) + trace_sched_stick_numa(p, env.src_cpu, env.best_cpu); + return ret; + } - return (&tg->list == &task_groups) ? NULL : tg; + ret = migrate_swap(p, env.best_task); + if (ret != 0) + trace_sched_stick_numa(p, env.src_cpu, task_cpu(env.best_task)); + put_task_struct(env.best_task); + return ret; } -/* Iterate over all cfs_rq in a cpu */ -#define for_each_cfs_rq(cfs_rq, tg, cpu) \ - for (tg = container_of(&task_groups, struct task_group, list); \ - ((tg = next_task_group(tg)) && (cfs_rq = tg->cfs_rq[cpu]));) - -static void reset_cfs_rq_hmp_stats(int cpu, int reset_cra) +/* Attempt to migrate a task to a CPU on the preferred node. */ +static void numa_migrate_preferred(struct task_struct *p) { - struct task_group *tg; - struct cfs_rq *cfs_rq; - - rcu_read_lock(); + unsigned long interval = HZ; - for_each_cfs_rq(cfs_rq, tg, cpu) - reset_hmp_stats(&cfs_rq->hmp_stats, reset_cra); - - rcu_read_unlock(); -} + /* This task has no NUMA fault statistics yet */ + if (unlikely(p->numa_preferred_nid == -1 || !p->numa_faults_memory)) + return; -#else /* CONFIG_CFS_BANDWIDTH */ + /* Periodically retry migrating the task to the preferred node */ + interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); + p->numa_migrate_retry = jiffies + interval; -static inline void reset_cfs_rq_hmp_stats(int cpu, int reset_cra) { } + /* Success if task is already running on preferred CPU */ + if (task_node(p) == p->numa_preferred_nid) + return; -#endif /* CONFIG_CFS_BANDWIDTH */ + /* Otherwise, try migrate to a CPU on the preferred node */ + task_numa_migrate(p); +} /* - * Return total number of tasks "eligible" to run on highest capacity cpu + * Find the nodes on which the workload is actively running. We do this by + * tracking the nodes from which NUMA hinting faults are triggered. This can + * be different from the set of nodes where the workload's memory is currently + * located. * - * This is simply nr_big_tasks for cpus which are not of max_capacity and - * (nr_running - nr_small_tasks) for cpus of max_capacity + * The bitmask is used to make smarter decisions on when to do NUMA page + * migrations, To prevent flip-flopping, and excessive page migrations, nodes + * are added when they cause over 6/16 of the maximum number of faults, but + * only removed when they drop below 3/16. */ -unsigned int nr_eligible_big_tasks(int cpu) +static void update_numa_active_node_mask(struct numa_group *numa_group) { - struct rq *rq = cpu_rq(cpu); - int nr_big = rq->hmp_stats.nr_big_tasks; - int nr = rq->nr_running; - int nr_small = rq->hmp_stats.nr_small_tasks; - - if (rq->max_possible_capacity != max_possible_capacity) - return nr_big; + unsigned long faults, max_faults = 0; + int nid; - /* Consider all (except small) tasks on max_capacity cpu as big tasks */ - nr_big = nr - nr_small; - if (nr_big < 0) - nr_big = 0; + for_each_online_node(nid) { + faults = group_faults_cpu(numa_group, nid); + if (faults > max_faults) + max_faults = faults; + } - return nr_big; + for_each_online_node(nid) { + faults = group_faults_cpu(numa_group, nid); + if (!node_isset(nid, numa_group->active_nodes)) { + if (faults > max_faults * 6 / 16) + node_set(nid, numa_group->active_nodes); + } else if (faults < max_faults * 3 / 16) + node_clear(nid, numa_group->active_nodes); + } } /* - * reset_cpu_hmp_stats - reset HMP stats for a cpu - * nr_big_tasks, nr_small_tasks - * cumulative_runnable_avg (iff reset_cra is true) + * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS + * increments. The more local the fault statistics are, the higher the scan + * period will be for the next scan window. If local/(local+remote) ratio is + * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) + * the scan period will decrease. Aim for 70% local accesses. + */ +#define NUMA_PERIOD_SLOTS 10 +#define NUMA_PERIOD_THRESHOLD 7 + +/* + * Increase the scan period (slow down scanning) if the majority of + * our memory is already on our local node, or if the majority of + * the page accesses are shared with other processes. + * Otherwise, decrease the scan period. */ -void reset_cpu_hmp_stats(int cpu, int reset_cra) +static void update_task_scan_period(struct task_struct *p, + unsigned long shared, unsigned long private) { - reset_cfs_rq_hmp_stats(cpu, reset_cra); - reset_hmp_stats(&cpu_rq(cpu)->hmp_stats, reset_cra); -} + unsigned int period_slot; + int ratio; + int diff; -#ifdef CONFIG_CFS_BANDWIDTH + unsigned long remote = p->numa_faults_locality[0]; + unsigned long local = p->numa_faults_locality[1]; -static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); + /* + * If there were no record hinting faults then either the task is + * completely idle or all activity is areas that are not of interest + * to automatic numa balancing. Scan slower + */ + if (local + shared == 0) { + p->numa_scan_period = min(p->numa_scan_period_max, + p->numa_scan_period << 1); -static void inc_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra); -static void dec_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra); + p->mm->numa_next_scan = jiffies + + msecs_to_jiffies(p->numa_scan_period); -/* Add task's contribution to a cpu' HMP statistics */ -static void -_inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p, int change_cra) -{ - struct cfs_rq *cfs_rq; - struct sched_entity *se = &p->se; + return; + } /* - * Although below check is not strictly required (as - * inc/dec_nr_big_small_task and inc/dec_cumulative_runnable_avg called - * from inc_cfs_rq_hmp_stats() have similar checks), we gain a bit on - * efficiency by short-circuiting for_each_sched_entity() loop when - * !sched_enable_hmp || sched_disable_window_stats + * Prepare to scale scan period relative to the current period. + * == NUMA_PERIOD_THRESHOLD scan period stays the same + * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) + * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) */ - if (!sched_enable_hmp || sched_disable_window_stats) - return; + period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); + ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); + if (ratio >= NUMA_PERIOD_THRESHOLD) { + int slot = ratio - NUMA_PERIOD_THRESHOLD; + if (!slot) + slot = 1; + diff = slot * period_slot; + } else { + diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - inc_cfs_rq_hmp_stats(cfs_rq, p, change_cra); - if (cfs_rq_throttled(cfs_rq)) - break; + /* + * Scale scan rate increases based on sharing. There is an + * inverse relationship between the degree of sharing and + * the adjustment made to the scanning period. Broadly + * speaking the intent is that there is little point + * scanning faster if shared accesses dominate as it may + * simply bounce migrations uselessly + */ + ratio = DIV_ROUND_UP(private * NUMA_PERIOD_SLOTS, (private + shared + 1)); + diff = (diff * ratio) / NUMA_PERIOD_SLOTS; } - /* Update rq->hmp_stats only if we didn't find any throttled cfs_rq */ - if (!se) - inc_rq_hmp_stats(rq, p, change_cra); + p->numa_scan_period = clamp(p->numa_scan_period + diff, + task_scan_min(p), task_scan_max(p)); + memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); } -/* Remove task's contribution from a cpu' HMP statistics */ -static void -_dec_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p, int change_cra) -{ - struct cfs_rq *cfs_rq; - struct sched_entity *se = &p->se; - - /* See comment on efficiency in _inc_hmp_sched_stats_fair */ - if (!sched_enable_hmp || sched_disable_window_stats) - return; - - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - dec_cfs_rq_hmp_stats(cfs_rq, p, change_cra); - if (cfs_rq_throttled(cfs_rq)) - break; +/* + * Get the fraction of time the task has been running since the last + * NUMA placement cycle. The scheduler keeps similar statistics, but + * decays those on a 32ms period, which is orders of magnitude off + * from the dozens-of-seconds NUMA balancing period. Use the scheduler + * stats only if the task is so new there are no NUMA statistics yet. + */ +static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) +{ + u64 runtime, delta, now; + /* Use the start of this time slice to avoid calculations. */ + now = p->se.exec_start; + runtime = p->se.sum_exec_runtime; + + if (p->last_task_numa_placement) { + delta = runtime - p->last_sum_exec_runtime; + *period = now - p->last_task_numa_placement; + } else { + delta = p->se.avg.runnable_avg_sum; + *period = p->se.avg.avg_period; } - /* Update rq->hmp_stats only if we didn't find any throttled cfs_rq */ - if (!se) - dec_rq_hmp_stats(rq, p, change_cra); -} + p->last_sum_exec_runtime = runtime; + p->last_task_numa_placement = now; -static void inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p) -{ - _inc_hmp_sched_stats_fair(rq, p, 1); + return delta; } -static void dec_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p) +static void task_numa_placement(struct task_struct *p) { - _dec_hmp_sched_stats_fair(rq, p, 1); -} + int seq, nid, max_nid = -1, max_group_nid = -1; + unsigned long max_faults = 0, max_group_faults = 0; + unsigned long fault_types[2] = { 0, 0 }; + unsigned long total_faults; + u64 runtime, period; + spinlock_t *group_lock = NULL; -static int task_will_be_throttled(struct task_struct *p); + seq = ACCESS_ONCE(p->mm->numa_scan_seq); + if (p->numa_scan_seq == seq) + return; + p->numa_scan_seq = seq; + p->numa_scan_period_max = task_scan_max(p); -#else /* CONFIG_CFS_BANDWIDTH */ + total_faults = p->numa_faults_locality[0] + + p->numa_faults_locality[1]; + runtime = numa_get_avg_runtime(p, &period); -static void -inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p) -{ - inc_nr_big_small_task(&rq->hmp_stats, p); - inc_cumulative_runnable_avg(&rq->hmp_stats, p); -} + /* If the task is part of a group prevent parallel updates to group stats */ + if (p->numa_group) { + group_lock = &p->numa_group->lock; + spin_lock_irq(group_lock); + } -static void -dec_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p) -{ - dec_nr_big_small_task(&rq->hmp_stats, p); - dec_cumulative_runnable_avg(&rq->hmp_stats, p); -} + /* Find the node with the highest number of faults */ + for_each_online_node(nid) { + unsigned long faults = 0, group_faults = 0; + int priv, i; -static inline int task_will_be_throttled(struct task_struct *p) -{ - return 0; -} + for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { + long diff, f_diff, f_weight; -static void -_inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p, int change_cra) -{ - inc_nr_big_small_task(&rq->hmp_stats, p); -} + i = task_faults_idx(nid, priv); -#endif /* CONFIG_CFS_BANDWIDTH */ + /* Decay existing window, copy faults since last scan */ + diff = p->numa_faults_buffer_memory[i] - p->numa_faults_memory[i] / 2; + fault_types[priv] += p->numa_faults_buffer_memory[i]; + p->numa_faults_buffer_memory[i] = 0; -/* - * Walk runqueue of cpu and re-initialize 'nr_big_tasks' and 'nr_small_tasks' - * counters. - */ -void fixup_nr_big_small_task(int cpu, int reset_stats) -{ - struct rq *rq = cpu_rq(cpu); - struct task_struct *p; + /* + * Normalize the faults_from, so all tasks in a group + * count according to CPU use, instead of by the raw + * number of faults. Tasks with little runtime have + * little over-all impact on throughput, and thus their + * faults are less important. + */ + f_weight = div64_u64(runtime << 16, period + 1); + f_weight = (f_weight * p->numa_faults_buffer_cpu[i]) / + (total_faults + 1); + f_diff = f_weight - p->numa_faults_cpu[i] / 2; + p->numa_faults_buffer_cpu[i] = 0; + + p->numa_faults_memory[i] += diff; + p->numa_faults_cpu[i] += f_diff; + faults += p->numa_faults_memory[i]; + p->total_numa_faults += diff; + if (p->numa_group) { + /* safe because we can only change our own group */ + p->numa_group->faults[i] += diff; + p->numa_group->faults_cpu[i] += f_diff; + p->numa_group->total_faults += diff; + group_faults += p->numa_group->faults[i]; + } + } - /* fixup_nr_big_small_task() is called from two functions. In one of - * them stats are already reset, don't waste time resetting them again - */ - if (reset_stats) { - /* Do not reset cumulative_runnable_avg */ - reset_cpu_hmp_stats(cpu, 0); + if (faults > max_faults) { + max_faults = faults; + max_nid = nid; + } + + if (group_faults > max_group_faults) { + max_group_faults = group_faults; + max_group_nid = nid; + } } - list_for_each_entry(p, &rq->cfs_tasks, se.group_node) - _inc_hmp_sched_stats_fair(rq, p, 0); -} + update_task_scan_period(p, fault_types[0], fault_types[1]); -/* Disable interrupts and grab runqueue lock of all cpus listed in @cpus */ -void pre_big_small_task_count_change(const struct cpumask *cpus) -{ - int i; + if (p->numa_group) { + update_numa_active_node_mask(p->numa_group); + spin_unlock_irq(group_lock); + max_nid = max_group_nid; + } - local_irq_disable(); + if (max_faults) { + /* Set the new preferred node */ + if (max_nid != p->numa_preferred_nid) + sched_setnuma(p, max_nid); - for_each_cpu(i, cpus) - raw_spin_lock(&cpu_rq(i)->lock); + if (task_node(p) != p->numa_preferred_nid) + numa_migrate_preferred(p); + } } -/* - * Reinitialize 'nr_big_tasks' and 'nr_small_tasks' counters on all affected - * cpus - */ -void post_big_small_task_count_change(const struct cpumask *cpus) +static inline int get_numa_group(struct numa_group *grp) { - int i; - - /* Assumes local_irq_disable() keeps online cpumap stable */ - for_each_cpu(i, cpus) - fixup_nr_big_small_task(i, 1); - - for_each_cpu(i, cpus) - raw_spin_unlock(&cpu_rq(i)->lock); - - local_irq_enable(); + return atomic_inc_not_zero(&grp->refcount); } -DEFINE_MUTEX(policy_mutex); - -#ifdef CONFIG_SCHED_FREQ_INPUT -static inline int invalid_value_freq_input(unsigned int *data) -{ - if (data == &sysctl_sched_migration_fixup) - return !(*data == 0 || *data == 1); - - if (data == &sysctl_sched_freq_account_wait_time) - return !(*data == 0 || *data == 1); - - return 0; -} -#else -static inline int invalid_value_freq_input(unsigned int *data) +static inline void put_numa_group(struct numa_group *grp) { - return 0; + if (atomic_dec_and_test(&grp->refcount)) + kfree_rcu(grp, rcu); } -#endif -static inline int invalid_value(unsigned int *data) +static void task_numa_group(struct task_struct *p, int cpupid, int flags, + int *priv) { - unsigned int val = *data; - - if (data == &sysctl_sched_ravg_hist_size) - return (val < 2 || val > RAVG_HIST_SIZE_MAX); - - if (data == &sysctl_sched_window_stats_policy) - return val >= WINDOW_STATS_INVALID_POLICY; - - if (data == &sysctl_sched_account_wait_time) - return !(val == 0 || val == 1); + struct numa_group *grp, *my_grp; + struct task_struct *tsk; + bool join = false; + int cpu = cpupid_to_cpu(cpupid); + int i; - return invalid_value_freq_input(data); -} + if (unlikely(!p->numa_group)) { + unsigned int size = sizeof(struct numa_group) + + 4*nr_node_ids*sizeof(unsigned long); -/* - * Handle "atomic" update of sysctl_sched_window_stats_policy, - * sysctl_sched_ravg_hist_size, sysctl_sched_account_wait_time and - * sched_freq_legacy_mode variables. - */ -int sched_window_update_handler(struct ctl_table *table, int write, - void __user *buffer, size_t *lenp, - loff_t *ppos) -{ - int ret; - unsigned int *data = (unsigned int *)table->data; - unsigned int old_val; + grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); + if (!grp) + return; - if (!sched_enable_hmp) - return -EINVAL; + atomic_set(&grp->refcount, 1); + spin_lock_init(&grp->lock); + INIT_LIST_HEAD(&grp->task_list); + grp->gid = p->pid; + /* Second half of the array tracks nids where faults happen */ + grp->faults_cpu = grp->faults + NR_NUMA_HINT_FAULT_TYPES * + nr_node_ids; - mutex_lock(&policy_mutex); + node_set(task_node(current), grp->active_nodes); - old_val = *data; + for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) + grp->faults[i] = p->numa_faults_memory[i]; - ret = proc_dointvec(table, write, buffer, lenp, ppos); - if (ret || !write || (write && (old_val == *data))) - goto done; + grp->total_faults = p->total_numa_faults; - if (invalid_value(data)) { - *data = old_val; - ret = -EINVAL; - goto done; + list_add(&p->numa_entry, &grp->task_list); + grp->nr_tasks++; + rcu_assign_pointer(p->numa_group, grp); } - reset_all_window_stats(0, 0); + rcu_read_lock(); + tsk = ACCESS_ONCE(cpu_rq(cpu)->curr); -done: - mutex_unlock(&policy_mutex); + if (!cpupid_match_pid(tsk, cpupid)) + goto no_join; - return ret; -} + grp = rcu_dereference(tsk->numa_group); + if (!grp) + goto no_join; -/* - * Convert percentage value into absolute form. This will avoid div() operation - * in fast path, to convert task load in percentage scale. - */ -int sched_hmp_proc_update_handler(struct ctl_table *table, int write, - void __user *buffer, size_t *lenp, - loff_t *ppos) -{ - int ret; - unsigned int old_val; - unsigned int *data = (unsigned int *)table->data; - int update_min_nice = 0; + my_grp = p->numa_group; + if (grp == my_grp) + goto no_join; - mutex_lock(&policy_mutex); + /* + * Only join the other group if its bigger; if we're the bigger group, + * the other task will join us. + */ + if (my_grp->nr_tasks > grp->nr_tasks) + goto no_join; - old_val = *data; + /* + * Tie-break on the grp address. + */ + if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) + goto no_join; - ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + /* Always join threads in the same process. */ + if (tsk->mm == current->mm) + join = true; - if (ret || !write || !sched_enable_hmp) - goto done; + /* Simple filter to avoid false positives due to PID collisions */ + if (flags & TNF_SHARED) + join = true; - if (write && (old_val == *data)) - goto done; + /* Update priv based on whether false sharing was detected */ + *priv = !join; - if (data == &sysctl_sched_min_runtime) { - sched_min_runtime = ((u64) sysctl_sched_min_runtime) * 1000; - goto done; - } + if (join && !get_numa_group(grp)) + goto no_join; - if (data == (unsigned int *)&sysctl_sched_upmigrate_min_nice) { - if ((*(int *)data) < -20 || (*(int *)data) > 19) { - *data = old_val; - ret = -EINVAL; - goto done; - } - update_min_nice = 1; - } else { - /* all tunables other than min_nice are in percentage */ - if (sysctl_sched_downmigrate_pct > - sysctl_sched_upmigrate_pct || *data > 100) { - *data = old_val; - ret = -EINVAL; - goto done; - } - } + rcu_read_unlock(); - /* - * Big/Small task tunable change will need to re-classify tasks on - * runqueue as big and small and set their counters appropriately. - * sysctl interface affects secondary variables (*_pct), which is then - * "atomically" carried over to the primary variables. Atomic change - * includes taking runqueue lock of all online cpus and re-initiatizing - * their big/small counter values based on changed criteria. - */ - if ((data == &sysctl_sched_upmigrate_pct || - data == &sysctl_sched_small_task_pct || update_min_nice)) { - get_online_cpus(); - pre_big_small_task_count_change(cpu_online_mask); - } + if (!join) + return; - set_hmp_defaults(); + BUG_ON(irqs_disabled()); + double_lock_irq(&my_grp->lock, &grp->lock); - if ((data == &sysctl_sched_upmigrate_pct || - data == &sysctl_sched_small_task_pct || update_min_nice)) { - post_big_small_task_count_change(cpu_online_mask); - put_online_cpus(); + for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { + my_grp->faults[i] -= p->numa_faults_memory[i]; + grp->faults[i] += p->numa_faults_memory[i]; } + my_grp->total_faults -= p->total_numa_faults; + grp->total_faults += p->total_numa_faults; -done: - mutex_unlock(&policy_mutex); - return ret; -} + list_move(&p->numa_entry, &grp->task_list); + my_grp->nr_tasks--; + grp->nr_tasks++; -/* - * Reset balance_interval at all sched_domain levels of given cpu, so that it - * honors kick. - */ -static inline void reset_balance_interval(int cpu) -{ - struct sched_domain *sd; + spin_unlock(&my_grp->lock); + spin_unlock_irq(&grp->lock); - if (cpu >= nr_cpu_ids) - return; + rcu_assign_pointer(p->numa_group, grp); - rcu_read_lock(); - for_each_domain(cpu, sd) - sd->balance_interval = 0; + put_numa_group(my_grp); + return; + +no_join: rcu_read_unlock(); + return; } -static inline int find_new_hmp_ilb(int call_cpu, int type) +void task_numa_free(struct task_struct *p) { + struct numa_group *grp = p->numa_group; + void *numa_faults = p->numa_faults_memory; + unsigned long flags; int i; - int best_cpu = nr_cpu_ids; - struct sched_domain *sd; - int min_cost = INT_MAX, cost; - struct rq *src_rq = cpu_rq(call_cpu); - struct rq *dst_rq; - - rcu_read_lock(); - /* Pick an idle cpu "closest" to call_cpu */ - for_each_domain(call_cpu, sd) { - for_each_cpu(i, sched_domain_span(sd)) { - dst_rq = cpu_rq(i); - if (!idle_cpu(i) || (type == NOHZ_KICK_RESTRICT - && dst_rq->capacity > src_rq->capacity)) - continue; - - cost = power_cost_at_freq(i, min_max_freq); - if (cost < min_cost) { - best_cpu = i; - min_cost = cost; - } - } + if (grp) { + spin_lock_irqsave(&grp->lock, flags); + for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) + grp->faults[i] -= p->numa_faults_memory[i]; + grp->total_faults -= p->total_numa_faults; - if (best_cpu < nr_cpu_ids) - break; + list_del(&p->numa_entry); + grp->nr_tasks--; + spin_unlock_irqrestore(&grp->lock, flags); + RCU_INIT_POINTER(p->numa_group, NULL); + put_numa_group(grp); } - rcu_read_unlock(); - - reset_balance_interval(best_cpu); - - return best_cpu; + p->numa_faults_memory = NULL; + p->numa_faults_buffer_memory = NULL; + p->numa_faults_cpu= NULL; + p->numa_faults_buffer_cpu = NULL; + kfree(numa_faults); } /* - * For the current task's CPU, we don't check whether there are - * multiple tasks. Just see if running the task on another CPU is - * lower power than running only this task on the current CPU. This is - * not the most accurate model, but we should be load balanced most of - * the time anyway. */ -static int lower_power_cpu_available(struct task_struct *p, int cpu) + * Got a PROT_NONE fault for a page on @node. + */ +void task_numa_fault(int mem_node, int pages, int migrated) { - int i; - int lowest_power_cpu = task_cpu(p); - int lowest_power = power_cost(scale_load_to_cpu(task_load(p), - lowest_power_cpu), lowest_power_cpu); - struct cpumask search_cpus; - struct rq *rq = cpu_rq(cpu); + struct task_struct *p = current; + + if (!sched_feat_numa(NUMA)) + return; + + /* FIXME: Allocate task-specific structure for placement policy here */ /* - * This function should be called only when task 'p' fits in the current - * CPU which can be ensured by task_will_fit() prior to this. + * If pages are properly placed (did not migrate) then scan slower. + * This is reset periodically in case of phase changes */ - cpumask_and(&search_cpus, tsk_cpus_allowed(p), cpu_online_mask); - cpumask_and(&search_cpus, &search_cpus, &rq->freq_domain_cpumask); - cpumask_clear_cpu(lowest_power_cpu, &search_cpus); - - /* Is a lower-powered idle CPU available which will fit this task? */ - for_each_cpu(i, &search_cpus) { - if (idle_cpu(i)) { - int cost = - power_cost(scale_load_to_cpu(task_load(p), i), i); - if (cost < lowest_power) { - lowest_power_cpu = i; - lowest_power = cost; - } - } - } + if (!migrated) + p->numa_scan_period = min(sysctl_numa_balancing_scan_period_max, + p->numa_scan_period + jiffies_to_msecs(10)); - return (lowest_power_cpu != task_cpu(p)); + task_numa_placement(p); } -static inline int is_task_migration_throttled(struct task_struct *p); +static void reset_ptenuma_scan(struct task_struct *p) +{ + ACCESS_ONCE(p->mm->numa_scan_seq)++; + p->mm->numa_scan_offset = 0; +} /* - * Check if a task is on the "wrong" cpu (i.e its current cpu is not the ideal - * cpu as per its demand or priority) - * - * Returns reason why task needs to be migrated + * The expensive part of numa migration is done from task_work context. + * Triggered from task_tick_numa(). */ -static inline int migration_needed(struct rq *rq, struct task_struct *p) +void task_numa_work(struct callback_head *work) { - int nice = TASK_NICE(p); + unsigned long migrate, next_scan, now = jiffies; + struct task_struct *p = current; + struct mm_struct *mm = p->mm; + struct vm_area_struct *vma; + unsigned long start, end; + unsigned long nr_pte_updates = 0; + long pages; - if (!sched_enable_hmp || p->state != TASK_RUNNING) - return 0; + WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work)); - /* No need to migrate task that is about to be throttled */ - if (task_will_be_throttled(p)) - return 0; + work->next = work; /* protect against double add */ + /* + * Who cares about NUMA placement when they're dying. + * + * NOTE: make sure not to dereference p->mm before this check, + * exit_task_work() happens _after_ exit_mm() so we could be called + * without p->mm even though we still had it when we enqueued this + * work. + */ + if (p->flags & PF_EXITING) + return; + + if (!mm->numa_next_scan) { + mm->numa_next_scan = now + + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); + } - if (sched_boost()) { - if (rq->capacity != max_capacity) - return UP_MIGRATION; + /* + * Enforce maximal scan/migration frequency.. + */ + migrate = mm->numa_next_scan; + if (time_before(now, migrate)) + return; - return 0; + if (p->numa_scan_period == 0) { + p->numa_scan_period_max = task_scan_max(p); + p->numa_scan_period = task_scan_min(p); } - if (is_small_task(p)) - return 0; + next_scan = now + msecs_to_jiffies(p->numa_scan_period); + if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate) + return; - if (sched_cpu_high_irqload(cpu_of(rq))) - return IRQLOAD_MIGRATION; + /* + * Delay this task enough that another task of this mm will likely win + * the next time around. + */ + p->node_stamp += 2 * TICK_NSEC; - if ((nice > sched_upmigrate_min_nice || upmigrate_discouraged(p)) && - rq->capacity > min_capacity) - return DOWN_MIGRATION; + start = mm->numa_scan_offset; + pages = sysctl_numa_balancing_scan_size; + pages <<= 20 - PAGE_SHIFT; /* MB in pages */ + if (!pages) + return; - if (!task_will_fit(p, cpu_of(rq))) - return UP_MIGRATION; + down_read(&mm->mmap_sem); + vma = find_vma(mm, start); + if (!vma) { + reset_ptenuma_scan(p); + start = 0; + vma = mm->mmap; + } + for (; vma; vma = vma->vm_next) { + if (!vma_migratable(vma) || !vma_policy_mof(vma) || + is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { + continue; + } - if (sysctl_sched_enable_power_aware && - !is_task_migration_throttled(p) && - is_cpu_throttling_imminent(cpu_of(rq)) && - lower_power_cpu_available(p, cpu_of(rq))) - return EA_MIGRATION; + /* + * Shared library pages mapped by multiple processes are not + * migrated as it is expected they are cache replicated. Avoid + * hinting faults in read-only file-backed mappings or the vdso + * as migrating the pages will be of marginal benefit. + */ + if (!vma->vm_mm || + (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) + continue; - return 0; -} + /* + * Skip inaccessible VMAs to avoid any confusion between + * PROT_NONE and NUMA hinting ptes + */ + if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) + continue; -static DEFINE_RAW_SPINLOCK(migration_lock); + do { + start = max(start, vma->vm_start); + end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); + end = min(end, vma->vm_end); + nr_pte_updates += change_prot_numa(vma, start, end); -static inline int -kick_active_balance(struct rq *rq, struct task_struct *p, int new_cpu) -{ - unsigned long flags; - int rc = 0; + /* + * Scan sysctl_numa_balancing_scan_size but ensure that + * at least one PTE is updated so that unused virtual + * address space is quickly skipped. + */ + if (nr_pte_updates) + pages -= (end - start) >> PAGE_SHIFT; - /* Invoke active balance to force migrate currently running task */ - raw_spin_lock_irqsave(&rq->lock, flags); - if (!rq->active_balance) { - rq->active_balance = 1; - rq->push_cpu = new_cpu; - get_task_struct(p); - rq->push_task = p; - rc = 1; + start = end; + if (pages <= 0) + goto out; + + cond_resched(); + } while (end != vma->vm_end); } - raw_spin_unlock_irqrestore(&rq->lock, flags); - return rc; +out: + /* + * It is possible to reach the end of the VMA list but the last few + * VMAs are not guaranteed to the vma_migratable. If they are not, we + * would find the !migratable VMA on the next scan but not reset the + * scanner to the start so check it now. + */ + if (vma) + mm->numa_scan_offset = start; + else + reset_ptenuma_scan(p); + up_read(&mm->mmap_sem); } /* - * Check if currently running task should be migrated to a better cpu. - * - * Todo: Effect this via changes to nohz_balancer_kick() and load balance? + * Drive the periodic memory faults.. */ -void check_for_migration(struct rq *rq, struct task_struct *p) +void task_tick_numa(struct rq *rq, struct task_struct *curr) { - int cpu = cpu_of(rq), new_cpu; - int active_balance = 0, reason; + struct callback_head *work = &curr->numa_work; + u64 period, now; - reason = migration_needed(rq, p); - if (!reason) + /* + * We don't care about NUMA placement if we don't have memory. + */ + if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work) return; - raw_spin_lock(&migration_lock); - new_cpu = select_best_cpu(p, cpu, reason, 0); - - if (new_cpu != cpu) { - active_balance = kick_active_balance(rq, p, new_cpu); - if (active_balance) - mark_reserved(new_cpu); - } + /* + * Using runtime rather than walltime has the dual advantage that + * we (mostly) drive the selection from busy threads and that the + * task needs to have done some actual work before we bother with + * NUMA placement. + */ + now = curr->se.sum_exec_runtime; + period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; - raw_spin_unlock(&migration_lock); + if (now - curr->node_stamp > period) { + if (!curr->node_stamp) + curr->numa_scan_period = task_scan_min(curr); + curr->node_stamp += period; - if (active_balance) - stop_one_cpu_nowait(cpu, active_load_balance_cpu_stop, rq, - &rq->active_balance_work); + if (!time_before(jiffies, curr->mm->numa_next_scan)) { + init_task_work(work, task_numa_work); /* TODO: move this into sched_fork() */ + task_work_add(curr, work, true); + } + } } - -static inline int nr_big_tasks(struct rq *rq) +#else +static void task_tick_numa(struct rq *rq, struct task_struct *curr) { - return rq->hmp_stats.nr_big_tasks; } -static inline int is_cpu_throttling_imminent(int cpu) +static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p) { - int throttling = 0; - struct cpu_pwr_stats *per_cpu_info; - - if (sched_feat(FORCE_CPU_THROTTLING_IMMINENT)) - return 1; +} - per_cpu_info = get_cpu_pwr_stats(); - if (per_cpu_info) - throttling = per_cpu_info[cpu].throttling; - return throttling; +static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p) +{ } +#endif /* CONFIG_NUMA_BALANCING */ -static inline int is_task_migration_throttled(struct task_struct *p) +static void +account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) { - u64 delta = sched_clock() - p->run_start; + update_load_add(&cfs_rq->load, se->load.weight); + if (!parent_entity(se)) + update_load_add(&rq_of(cfs_rq)->load, se->load.weight); +#ifdef CONFIG_SMP + if (entity_is_task(se)) { + struct rq *rq = rq_of(cfs_rq); - return delta < sched_min_runtime; + account_numa_enqueue(rq, task_of(se)); + list_add(&se->group_node, &rq->cfs_tasks); + } +#endif + cfs_rq->nr_running++; } -unsigned int cpu_temp(int cpu) +static void +account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) { - struct cpu_pwr_stats *per_cpu_info = get_cpu_pwr_stats(); - if (per_cpu_info) - return per_cpu_info[cpu].temp; - else - return 0; + update_load_sub(&cfs_rq->load, se->load.weight); + if (!parent_entity(se)) + update_load_sub(&rq_of(cfs_rq)->load, se->load.weight); + if (entity_is_task(se)) { + account_numa_dequeue(rq_of(cfs_rq), task_of(se)); + list_del_init(&se->group_node); + } + cfs_rq->nr_running--; } -#else /* CONFIG_SCHED_HMP */ +#ifdef CONFIG_FAIR_GROUP_SCHED +# ifdef CONFIG_SMP +static inline long calc_tg_weight(struct task_group *tg, struct cfs_rq *cfs_rq) +{ + long tg_weight; -#define sysctl_sched_enable_power_aware 0 + /* + * Use this CPU's actual weight instead of the last load_contribution + * to gain a more accurate current total weight. See + * update_cfs_rq_load_contribution(). + */ + tg_weight = atomic_long_read(&tg->load_avg); + tg_weight -= cfs_rq->tg_load_contrib; + tg_weight += cfs_rq->load.weight; -static inline int task_will_fit(struct task_struct *p, int cpu) -{ - return 1; + return tg_weight; } -static inline int select_best_cpu(struct task_struct *p, int target, - int reason, int sync) +static long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg) { - return 0; -} + long tg_weight, load, shares; -static inline int find_new_hmp_ilb(int call_cpu, int type) -{ - return 0; -} + tg_weight = calc_tg_weight(tg, cfs_rq); + load = cfs_rq->load.weight; -static inline int power_cost(u64 task_load, int cpu) -{ - return SCHED_CAPACITY_SCALE; -} + shares = (tg->shares * load); + if (tg_weight) + shares /= tg_weight; -static inline int -spill_threshold_crossed(u64 task_load, u64 cpu_load, struct rq *rq) -{ - return 0; -} + if (shares < MIN_SHARES) + shares = MIN_SHARES; + if (shares > tg->shares) + shares = tg->shares; -static inline int mostly_idle_cpu(int cpu) -{ - return 0; + return shares; } - -static inline int sched_boost(void) +# else /* CONFIG_SMP */ +static inline long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg) { - return 0; + return tg->shares; } - -static inline int is_small_task(struct task_struct *p) +# endif /* CONFIG_SMP */ +static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight) { - return 0; -} + if (se->on_rq) { + /* commit outstanding execution time */ + if (cfs_rq->curr == se) + update_curr(cfs_rq); + account_entity_dequeue(cfs_rq, se); + } -static inline int is_big_task(struct task_struct *p) -{ - return 0; -} + update_load_set(&se->load, weight); -static inline int nr_big_tasks(struct rq *rq) -{ - return 0; + if (se->on_rq) + account_entity_enqueue(cfs_rq, se); } -static inline int is_cpu_throttling_imminent(int cpu) -{ - return 0; -} +static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); -static inline int is_task_migration_throttled(struct task_struct *p) +static void update_cfs_shares(struct cfs_rq *cfs_rq) { - return 0; -} + struct task_group *tg; + struct sched_entity *se; + long shares; + + tg = cfs_rq->tg; + se = tg->se[cpu_of(rq_of(cfs_rq))]; + if (!se || throttled_hierarchy(cfs_rq)) + return; +#ifndef CONFIG_SMP + if (likely(se->load.weight == tg->shares)) + return; +#endif + shares = calc_cfs_shares(cfs_rq, tg); -unsigned int cpu_temp(int cpu) + reweight_entity(cfs_rq_of(se), se, shares); +} +#else /* CONFIG_FAIR_GROUP_SCHED */ +static inline void update_cfs_shares(struct cfs_rq *cfs_rq) { - return 0; } +#endif /* CONFIG_FAIR_GROUP_SCHED */ -static inline void -inc_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) { } -static inline void -dec_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) { } - -static inline void -inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p) { } - -static inline void -dec_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p) { } +#ifdef CONFIG_SMP +/* + * We choose a half-life close to 1 scheduling period. + * Note: The tables below are dependent on this value. + */ +#define LOAD_AVG_PERIOD 32 +#define LOAD_AVG_MAX 47742 /* maximum possible load avg */ +#define LOAD_AVG_MAX_N 345 /* number of full periods to produce LOAD_MAX_AVG */ -#endif /* CONFIG_SCHED_HMP */ +/* Precomputed fixed inverse multiplies for multiplication by y^n */ +static const u32 runnable_avg_yN_inv[] = { + 0xffffffff, 0xfa83b2da, 0xf5257d14, 0xefe4b99a, 0xeac0c6e6, 0xe5b906e6, + 0xe0ccdeeb, 0xdbfbb796, 0xd744fcc9, 0xd2a81d91, 0xce248c14, 0xc9b9bd85, + 0xc5672a10, 0xc12c4cc9, 0xbd08a39e, 0xb8fbaf46, 0xb504f333, 0xb123f581, + 0xad583ee9, 0xa9a15ab4, 0xa5fed6a9, 0xa2704302, 0x9ef5325f, 0x9b8d39b9, + 0x9837f050, 0x94f4efa8, 0x91c3d373, 0x8ea4398a, 0x8b95c1e3, 0x88980e80, + 0x85aac367, 0x82cd8698, +}; -#ifdef CONFIG_SCHED_HMP +/* + * Precomputed \Sum y^k { 1<=k<=n }. These are floor(true_value) to prevent + * over-estimates when re-combining. + */ +static const u32 runnable_avg_yN_sum[] = { + 0, 1002, 1982, 2941, 3880, 4798, 5697, 6576, 7437, 8279, 9103, + 9909,10698,11470,12226,12966,13690,14398,15091,15769,16433,17082, + 17718,18340,18949,19545,20128,20698,21256,21802,22336,22859,23371, +}; -void init_new_task_load(struct task_struct *p) +/* + * Approximate: + * val * y^n, where y^32 ~= 0.5 (~1 scheduling period) + */ +static __always_inline u64 decay_load(u64 val, u64 n) { - int i; - u32 init_load_windows = sched_init_task_load_windows; - u32 init_load_pelt = sched_init_task_load_pelt; - u32 init_load_pct = current->init_load_pct; + unsigned int local_n; + + if (!n) + return val; + else if (unlikely(n > LOAD_AVG_PERIOD * 63)) + return 0; - /* Note: child's init_load_pct itself would be 0 */ - memset(&p->ravg, 0, sizeof(struct ravg)); - p->se.avg.decay_count = 0; + /* after bounds checking we can collapse to 32-bit */ + local_n = n; - if (init_load_pct) { - init_load_pelt = div64_u64((u64)init_load_pct * - (u64)LOAD_AVG_MAX, 100); - init_load_windows = div64_u64((u64)init_load_pct * - (u64)sched_ravg_window, 100); + /* + * As y^PERIOD = 1/2, we can combine + * y^n = 1/2^(n/PERIOD) * y^(n%PERIOD) + * With a look-up table which covers y^n (n= LOAD_AVG_PERIOD)) { + val >>= local_n / LOAD_AVG_PERIOD; + local_n %= LOAD_AVG_PERIOD; } - p->ravg.demand = init_load_windows; - for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i) - p->ravg.sum_history[i] = init_load_windows; - - p->se.avg.runnable_avg_period = - init_load_pelt ? LOAD_AVG_MAX : 0; - p->se.avg.runnable_avg_sum = init_load_pelt; - p->se.avg.runnable_avg_sum_scaled = init_load_pelt; + val *= runnable_avg_yN_inv[local_n]; + /* We don't use SRR here since we always want to round down. */ + return val >> 32; } -#else /* CONFIG_SCHED_HMP */ - -#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED) - -void init_new_task_load(struct task_struct *p) +/* + * For updates fully spanning n periods, the contribution to runnable + * average will be: \Sum 1024*y^n + * + * We can compute this reasonably efficiently by combining: + * y^PERIOD = 1/2 with precomputed \Sum 1024*y^n {for n se.avg.decay_count = 0; - p->se.avg.runnable_avg_period = 0; - p->se.avg.runnable_avg_sum = 0; -} + u32 contrib = 0; -#else /* CONFIG_SMP && CONFIG_FAIR_GROUP_SCHED */ + if (likely(n <= LOAD_AVG_PERIOD)) + return runnable_avg_yN_sum[n]; + else if (unlikely(n >= LOAD_AVG_MAX_N)) + return LOAD_AVG_MAX; -void init_new_task_load(struct task_struct *p) -{ -} + /* Compute \Sum k^n combining precomputed values for k^i, \Sum k^j */ + do { + contrib /= 2; /* y^LOAD_AVG_PERIOD = 1/2 */ + contrib += runnable_avg_yN_sum[LOAD_AVG_PERIOD]; -#endif /* CONFIG_SMP && CONFIG_FAIR_GROUP_SCHED */ + n -= LOAD_AVG_PERIOD; + } while (n > LOAD_AVG_PERIOD); -#endif /* CONFIG_SCHED_HMP */ + contrib = decay_load(contrib, n); + return contrib + runnable_avg_yN_sum[n]; +} /* * We can represent the historical contribution to runnable average as the @@ -3106,13 +2283,19 @@ void init_new_task_load(struct task_struct *p) * load_avg = u_0` + y*(u_0 + u_1*y + u_2*y^2 + ... ) * = u_0 + u_1*y + u_2*y^2 + ... [re-labeling u_i --> u_{i+1}] */ -static __always_inline int __update_entity_runnable_avg(int cpu, u64 now, +static __always_inline int __update_entity_runnable_avg(u64 now, int cpu, struct sched_avg *sa, - int runnable) + int runnable, + int running) { - u64 delta, periods; - u32 runnable_contrib; - int delta_w, decayed = 0; + u64 delta, scaled_delta, periods; + u32 runnable_contrib, scaled_runnable_contrib; + int delta_w, scaled_delta_w, decayed = 0; + unsigned long scale_freq = arch_scale_freq_capacity(NULL, cpu); + unsigned long scale_cpu = arch_scale_cpu_capacity(NULL, cpu); + + //TJK fix tracing: trace_sched_contrib_scale_f(cpu, scale_freq, scale_cpu); + delta = now - sa->last_runnable_update; /* @@ -3134,7 +2317,7 @@ static __always_inline int __update_entity_runnable_avg(int cpu, u64 now, sa->last_runnable_update = now; /* delta_w is the amount already accumulated against our next period */ - delta_w = sa->runnable_avg_period % 1024; + delta_w = sa->avg_period % 1024; if (delta + delta_w >= 1024) { /* period roll-over */ decayed = 1; @@ -3145,11 +2328,17 @@ static __always_inline int __update_entity_runnable_avg(int cpu, u64 now, * period and accrue it. */ delta_w = 1024 - delta_w; - if (runnable) { - sa->runnable_avg_sum += delta_w; - add_to_scaled_stat(cpu, sa, delta_w); - } - sa->runnable_avg_period += delta_w; + scaled_delta_w = (delta_w * scale_freq) >> SCHED_CAPACITY_SHIFT; + + if (runnable) + sa->runnable_avg_sum += scaled_delta_w; + + scaled_delta_w *= scale_cpu; + scaled_delta_w >>= SCHED_CAPACITY_SHIFT; + + if (running) + sa->running_avg_sum += scaled_delta_w; + sa->avg_period += delta_w; delta -= delta_w; @@ -3159,25 +2348,39 @@ static __always_inline int __update_entity_runnable_avg(int cpu, u64 now, sa->runnable_avg_sum = decay_load(sa->runnable_avg_sum, periods + 1); - decay_scaled_stat(sa, periods + 1); - sa->runnable_avg_period = decay_load(sa->runnable_avg_period, + sa->running_avg_sum = decay_load(sa->running_avg_sum, + periods + 1); + sa->avg_period = decay_load(sa->avg_period, periods + 1); /* Efficiently calculate \sum (1..n_period) 1024*y^i */ runnable_contrib = __compute_runnable_contrib(periods); - if (runnable) { - sa->runnable_avg_sum += runnable_contrib; - add_to_scaled_stat(cpu, sa, runnable_contrib); - } - sa->runnable_avg_period += runnable_contrib; + scaled_runnable_contrib = (runnable_contrib * scale_freq) + >> SCHED_CAPACITY_SHIFT; + + if (runnable) + sa->runnable_avg_sum += scaled_runnable_contrib; + + scaled_runnable_contrib *= scale_cpu; + scaled_runnable_contrib >>= SCHED_CAPACITY_SHIFT; + + if (running) + sa->running_avg_sum += scaled_runnable_contrib; + sa->avg_period += runnable_contrib; } /* Remainder of delta accrued against u_0` */ - if (runnable) { - sa->runnable_avg_sum += delta; - add_to_scaled_stat(cpu, sa, delta); - } - sa->runnable_avg_period += delta; + scaled_delta = (delta * scale_freq) >> SCHED_CAPACITY_SHIFT; + + if (runnable) + sa->runnable_avg_sum += scaled_delta; + + scaled_delta *= scale_cpu; + scaled_delta >>= SCHED_CAPACITY_SHIFT; + + if (running) + sa->running_avg_sum += scaled_delta; + sa->avg_period += delta; return decayed; } @@ -3189,13 +2392,13 @@ static inline u64 __synchronize_entity_decay(struct sched_entity *se) u64 decays = atomic64_read(&cfs_rq->decay_counter); decays -= se->avg.decay_count; - if (!decays) { - se->avg.decay_count = 0; + se->avg.decay_count = 0; + if (!decays) return 0; - } se->avg.load_avg_contrib = decay_load(se->avg.load_avg_contrib, decays); - se->avg.decay_count = 0; + se->avg.utilization_avg_contrib = + decay_load(se->avg.utilization_avg_contrib, decays); return decays; } @@ -3205,13 +2408,16 @@ static inline void __update_cfs_rq_tg_load_contrib(struct cfs_rq *cfs_rq, int force_update) { struct task_group *tg = cfs_rq->tg; - s64 tg_contrib; + long tg_contrib; tg_contrib = cfs_rq->runnable_load_avg + cfs_rq->blocked_load_avg; tg_contrib -= cfs_rq->tg_load_contrib; - if (force_update || abs64(tg_contrib) > cfs_rq->tg_load_contrib / 8) { - atomic64_add(tg_contrib, &tg->load_avg); + if (!tg_contrib) + return; + + if (force_update || abs(tg_contrib) > cfs_rq->tg_load_contrib / 8) { + atomic_long_add(tg_contrib, &tg->load_avg); cfs_rq->tg_load_contrib += tg_contrib; } } @@ -3227,8 +2433,8 @@ static inline void __update_tg_runnable_avg(struct sched_avg *sa, long contrib; /* The fraction of a cpu used by this cfs_rq */ - contrib = div_u64(sa->runnable_avg_sum << NICE_0_SHIFT, - sa->runnable_avg_period + 1); + contrib = div_u64((u64)sa->runnable_avg_sum << NICE_0_SHIFT, + sa->avg_period + 1); contrib -= cfs_rq->tg_runnable_contrib; if (abs(contrib) > cfs_rq->tg_runnable_contrib / 64) { @@ -3246,8 +2452,8 @@ static inline void __update_group_entity_contrib(struct sched_entity *se) u64 contrib; contrib = cfs_rq->tg_load_contrib * tg->shares; - se->avg.load_avg_contrib = div64_u64(contrib, - atomic64_read(&tg->load_avg) + 1); + se->avg.load_avg_contrib = div_u64(contrib, + atomic_long_read(&tg->load_avg) + 1); /* * For group entities we need to compute a correction term in the case @@ -3278,13 +2484,21 @@ static inline void __update_group_entity_contrib(struct sched_entity *se) se->avg.load_avg_contrib >>= NICE_0_SHIFT; } } -#else + +static inline void update_rq_runnable_avg(struct rq *rq, int runnable) +{ + __update_entity_runnable_avg(rq_clock_task(rq), cpu_of(rq), &rq->avg, + runnable, runnable); + __update_tg_runnable_avg(&rq->avg, &rq->cfs); +} +#else /* CONFIG_FAIR_GROUP_SCHED */ static inline void __update_cfs_rq_tg_load_contrib(struct cfs_rq *cfs_rq, int force_update) {} static inline void __update_tg_runnable_avg(struct sched_avg *sa, struct cfs_rq *cfs_rq) {} static inline void __update_group_entity_contrib(struct sched_entity *se) {} -#endif +static inline void update_rq_runnable_avg(struct rq *rq, int runnable) {} +#endif /* CONFIG_FAIR_GROUP_SCHED */ static inline void __update_task_entity_contrib(struct sched_entity *se) { @@ -3292,7 +2506,7 @@ static inline void __update_task_entity_contrib(struct sched_entity *se) /* avoid overflowing a 32-bit type w/ SCHED_LOAD_SCALE */ contrib = se->avg.runnable_avg_sum * scale_load_down(se->load.weight); - contrib /= (se->avg.runnable_avg_period + 1); + contrib /= (se->avg.avg_period + 1); se->avg.load_avg_contrib = scale_load(contrib); } @@ -3311,6 +2525,31 @@ static long __update_entity_load_avg_contrib(struct sched_entity *se) return se->avg.load_avg_contrib - old_contrib; } + +static inline void __update_task_entity_utilization(struct sched_entity *se) +{ + u32 contrib; + + /* avoid overflowing a 32-bit type w/ SCHED_LOAD_SCALE */ + contrib = se->avg.running_avg_sum * scale_load_down(SCHED_LOAD_SCALE); + contrib /= (se->avg.avg_period + 1); + se->avg.utilization_avg_contrib = scale_load(contrib); +} + +static long __update_entity_utilization_avg_contrib(struct sched_entity *se) +{ + long old_contrib = se->avg.utilization_avg_contrib; + + if (entity_is_task(se)) + __update_task_entity_utilization(se); + else + se->avg.utilization_avg_contrib = + group_cfs_rq(se)->utilization_load_avg + + group_cfs_rq(se)->utilization_blocked_avg; + + return se->avg.utilization_avg_contrib - old_contrib; +} + static inline void subtract_blocked_load_contrib(struct cfs_rq *cfs_rq, long load_contrib) { @@ -3320,6 +2559,15 @@ static inline void subtract_blocked_load_contrib(struct cfs_rq *cfs_rq, cfs_rq->blocked_load_avg = 0; } +static inline void subtract_utilization_blocked_contrib(struct cfs_rq *cfs_rq, + long utilization_contrib) +{ + if (likely(utilization_contrib < cfs_rq->utilization_blocked_avg)) + cfs_rq->utilization_blocked_avg -= utilization_contrib; + else + cfs_rq->utilization_blocked_avg = 0; +} + static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq); /* Update a sched_entity's runnable average */ @@ -3327,38 +2575,44 @@ static inline void update_entity_load_avg(struct sched_entity *se, int update_cfs_rq) { struct cfs_rq *cfs_rq = cfs_rq_of(se); - long contrib_delta; - u64 now; + long contrib_delta, utilization_delta; int cpu = cpu_of(rq_of(cfs_rq)); - int decayed; + u64 now; /* * For a group entity we need to use their owned cfs_rq_clock_task() in * case they are the parent of a throttled hierarchy. */ - if (entity_is_task(se)) { + if (entity_is_task(se)) now = cfs_rq_clock_task(cfs_rq); - if (se->on_rq) - dec_hmp_sched_stats_fair(rq_of(cfs_rq), task_of(se)); - } else + else now = cfs_rq_clock_task(group_cfs_rq(se)); - decayed = __update_entity_runnable_avg(cpu, now, &se->avg, se->on_rq); - if (entity_is_task(se) && se->on_rq) - inc_hmp_sched_stats_fair(rq_of(cfs_rq), task_of(se)); - - if (!decayed) + if (!__update_entity_runnable_avg(now, cpu, &se->avg, se->on_rq, + cfs_rq->curr == se)) return; contrib_delta = __update_entity_load_avg_contrib(se); + utilization_delta = __update_entity_utilization_avg_contrib(se); +#if 0 + // TJK: fix tracing + if (entity_is_task(se)) + trace_sched_load_avg_task(task_of(se), &se->avg); +#endif if (!update_cfs_rq) return; - if (se->on_rq) + if (se->on_rq) { cfs_rq->runnable_load_avg += contrib_delta; - else + cfs_rq->utilization_load_avg += utilization_delta; + } else { subtract_blocked_load_contrib(cfs_rq, -contrib_delta); + subtract_utilization_blocked_contrib(cfs_rq, + -utilization_delta); + } + + // TJK: fix tracing: trace_sched_load_avg_cpu(cpu, cfs_rq); } /* @@ -3374,14 +2628,21 @@ static void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq, int force_update) if (!decays && !force_update) return; - if (atomic64_read(&cfs_rq->removed_load)) { - u64 removed_load = atomic64_xchg(&cfs_rq->removed_load, 0); + if (atomic_long_read(&cfs_rq->removed_load)) { + unsigned long removed_load, removed_utilization; + removed_load = atomic_long_xchg(&cfs_rq->removed_load, 0); + removed_utilization = + atomic_long_xchg(&cfs_rq->removed_utilization, 0); subtract_blocked_load_contrib(cfs_rq, removed_load); + subtract_utilization_blocked_contrib(cfs_rq, + removed_utilization); } if (decays) { cfs_rq->blocked_load_avg = decay_load(cfs_rq->blocked_load_avg, decays); + cfs_rq->utilization_blocked_avg = + decay_load(cfs_rq->utilization_blocked_avg, decays); atomic64_add(decays, &cfs_rq->decay_counter); cfs_rq->last_decay = now; } @@ -3389,13 +2650,6 @@ static void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq, int force_update) __update_cfs_rq_tg_load_contrib(cfs_rq, force_update); } -static inline void update_rq_runnable_avg(struct rq *rq, int runnable) -{ - __update_entity_runnable_avg(cpu_of(rq), rq->clock_task, - &rq->avg, runnable); - __update_tg_runnable_avg(&rq->avg, &rq->cfs); -} - /* Add the load generated by se into cfs_rq's child load-average */ static inline void enqueue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, @@ -3405,9 +2659,13 @@ static inline void enqueue_entity_load_avg(struct cfs_rq *cfs_rq, * We track migrations using entity decay_count <= 0, on a wake-up * migration we use a negative decay count to track the remote decays * accumulated while sleeping. + * + * Newly forked tasks are enqueued with se->avg.decay_count == 0, they + * are seen by enqueue_entity_load_avg() as a migration with an already + * constructed load_avg_contrib. */ if (unlikely(se->avg.decay_count <= 0)) { - se->avg.last_runnable_update = rq_of(cfs_rq)->clock_task; + se->avg.last_runnable_update = rq_clock_task(rq_of(cfs_rq)); if (se->avg.decay_count) { /* * In a wake-up migration we have to approximate the @@ -3431,10 +2689,13 @@ static inline void enqueue_entity_load_avg(struct cfs_rq *cfs_rq, /* migrated tasks did not contribute to our blocked load */ if (wakeup) { subtract_blocked_load_contrib(cfs_rq, se->avg.load_avg_contrib); + subtract_utilization_blocked_contrib(cfs_rq, + se->avg.utilization_avg_contrib); update_entity_load_avg(se, 0); } cfs_rq->runnable_load_avg += se->avg.load_avg_contrib; + cfs_rq->utilization_load_avg += se->avg.utilization_avg_contrib; /* we force update consideration on load-balancer moves */ update_cfs_rq_blocked_load(cfs_rq, !wakeup); } @@ -3453,8 +2714,11 @@ static inline void dequeue_entity_load_avg(struct cfs_rq *cfs_rq, update_cfs_rq_blocked_load(cfs_rq, !sleep); cfs_rq->runnable_load_avg -= se->avg.load_avg_contrib; + cfs_rq->utilization_load_avg -= se->avg.utilization_avg_contrib; if (sleep) { cfs_rq->blocked_load_avg += se->avg.load_avg_contrib; + cfs_rq->utilization_blocked_avg += + se->avg.utilization_avg_contrib; se->avg.decay_count = atomic64_read(&cfs_rq->decay_counter); } /* migrations, e.g. sleep=0 leave decay_count == 0 */ } @@ -3479,7 +2743,10 @@ void idle_exit_fair(struct rq *this_rq) update_rq_runnable_avg(this_rq, 0); } -#else +static int idle_balance(struct rq *this_rq); + +#else /* CONFIG_SMP */ + static inline void update_entity_load_avg(struct sched_entity *se, int update_cfs_rq) {} static inline void update_rq_runnable_avg(struct rq *rq, int runnable) {} @@ -3492,149 +2759,12 @@ static inline void dequeue_entity_load_avg(struct cfs_rq *cfs_rq, static inline void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq, int force_update) {} -static inline void -inc_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) { } -static inline void -dec_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) { } - -#endif - -#ifdef CONFIG_SCHED_HMP - -/* Return task demand in percentage scale */ -unsigned int pct_task_load(struct task_struct *p) -{ - unsigned int load; - - load = div64_u64((u64)task_load(p) * 100, (u64)max_task_load()); - - return load; -} - -/* - * Add scaled version of 'delta' to runnable_avg_sum_scaled - * 'delta' is scaled in reference to "best" cpu - */ -static inline void -add_to_scaled_stat(int cpu, struct sched_avg *sa, u64 delta) -{ - struct rq *rq = cpu_rq(cpu); - int cur_freq = rq->cur_freq, max_freq = rq->max_freq; - int cpu_max_possible_freq = rq->max_possible_freq; - u64 scaled_delta; - int sf; - - if (!sched_enable_hmp) - return; - - if (unlikely(cur_freq > max_possible_freq || - (cur_freq == max_freq && - max_freq < cpu_max_possible_freq))) - cur_freq = max_possible_freq; - - scaled_delta = div64_u64(delta * cur_freq, max_possible_freq); - sf = (rq->efficiency * 1024) / max_possible_efficiency; - scaled_delta *= sf; - scaled_delta >>= 10; - sa->runnable_avg_sum_scaled += scaled_delta; -} - -static inline void decay_scaled_stat(struct sched_avg *sa, u64 periods) -{ - if (!sched_enable_hmp) - return; - - sa->runnable_avg_sum_scaled = - decay_load(sa->runnable_avg_sum_scaled, - periods); -} - -#ifdef CONFIG_CFS_BANDWIDTH - -static void init_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq) -{ - cfs_rq->hmp_stats.nr_big_tasks = 0; - cfs_rq->hmp_stats.nr_small_tasks = 0; - cfs_rq->hmp_stats.cumulative_runnable_avg = 0; -} - -static void inc_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra) -{ - inc_nr_big_small_task(&cfs_rq->hmp_stats, p); - if (change_cra) - inc_cumulative_runnable_avg(&cfs_rq->hmp_stats, p); -} - -static void dec_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra) -{ - dec_nr_big_small_task(&cfs_rq->hmp_stats, p); - if (change_cra) - dec_cumulative_runnable_avg(&cfs_rq->hmp_stats, p); -} - -static void inc_throttled_cfs_rq_hmp_stats(struct hmp_sched_stats *stats, - struct cfs_rq *cfs_rq) -{ - stats->nr_big_tasks += cfs_rq->hmp_stats.nr_big_tasks; - stats->nr_small_tasks += cfs_rq->hmp_stats.nr_small_tasks; - stats->cumulative_runnable_avg += - cfs_rq->hmp_stats.cumulative_runnable_avg; -} - -static void dec_throttled_cfs_rq_hmp_stats(struct hmp_sched_stats *stats, - struct cfs_rq *cfs_rq) -{ - stats->nr_big_tasks -= cfs_rq->hmp_stats.nr_big_tasks; - stats->nr_small_tasks -= cfs_rq->hmp_stats.nr_small_tasks; - stats->cumulative_runnable_avg -= - cfs_rq->hmp_stats.cumulative_runnable_avg; - - BUG_ON(stats->nr_big_tasks < 0 || stats->nr_small_tasks < 0 || - (s64)stats->cumulative_runnable_avg < 0); -} - -#else /* CONFIG_CFS_BANDWIDTH */ - -static inline void inc_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra) { } - -static inline void dec_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra) { } - -#endif /* CONFIG_CFS_BANDWIDTH */ - -#else /* CONFIG_SCHED_HMP */ - -static inline void -add_to_scaled_stat(int cpu, struct sched_avg *sa, u64 delta) -{ -} - -static inline void decay_scaled_stat(struct sched_avg *sa, u64 periods) -{ -} - -static inline void init_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq) { } - -static inline void inc_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra) { } - -static inline void dec_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra) { } - -static inline void inc_throttled_cfs_rq_hmp_stats(struct hmp_sched_stats *stats, - struct cfs_rq *cfs_rq) -{ -} - -static inline void dec_throttled_cfs_rq_hmp_stats(struct hmp_sched_stats *stats, - struct cfs_rq *cfs_rq) +static inline int idle_balance(struct rq *rq) { + return 0; } -#endif /* CONFIG_SCHED_HMP */ +#endif /* CONFIG_SMP */ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) { @@ -3645,7 +2775,7 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) tsk = task_of(se); if (se->statistics.sleep_start) { - u64 delta = rq_of(cfs_rq)->clock - se->statistics.sleep_start; + u64 delta = rq_clock(rq_of(cfs_rq)) - se->statistics.sleep_start; if ((s64)delta < 0) delta = 0; @@ -3662,7 +2792,7 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) } } if (se->statistics.block_start) { - u64 delta = rq_of(cfs_rq)->clock - se->statistics.block_start; + u64 delta = rq_clock(rq_of(cfs_rq)) - se->statistics.block_start; if ((s64)delta < 0) delta = 0; @@ -3751,7 +2881,7 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { /* * Update the normalized vruntime before updating min_vruntime - * through callig update_curr(). + * through calling update_curr(). */ if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_WAKING)) se->vruntime += cfs_rq->min_vruntime; @@ -3785,10 +2915,10 @@ static void __clear_buddies_last(struct sched_entity *se) { for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (cfs_rq->last == se) - cfs_rq->last = NULL; - else + if (cfs_rq->last != se) break; + + cfs_rq->last = NULL; } } @@ -3796,10 +2926,10 @@ static void __clear_buddies_next(struct sched_entity *se) { for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (cfs_rq->next == se) - cfs_rq->next = NULL; - else + if (cfs_rq->next != se) break; + + cfs_rq->next = NULL; } } @@ -3807,10 +2937,10 @@ static void __clear_buddies_skip(struct sched_entity *se) { for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (cfs_rq->skip == se) - cfs_rq->skip = NULL; - else + if (cfs_rq->skip != se) break; + + cfs_rq->skip = NULL; } } @@ -3835,6 +2965,8 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) * Update run-time statistics of the 'current'. */ update_curr(cfs_rq); + if (entity_is_task(se) && task_of(se)->state == TASK_DEAD) + flags &= !DEQUEUE_SLEEP; dequeue_entity_load_avg(cfs_rq, se, flags & DEQUEUE_SLEEP); update_stats_dequeue(cfs_rq, se); @@ -3844,9 +2976,9 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) struct task_struct *tsk = task_of(se); if (tsk->state & TASK_INTERRUPTIBLE) - se->statistics.sleep_start = rq_of(cfs_rq)->clock; + se->statistics.sleep_start = rq_clock(rq_of(cfs_rq)); if (tsk->state & TASK_UNINTERRUPTIBLE) - se->statistics.block_start = rq_of(cfs_rq)->clock; + se->statistics.block_start = rq_clock(rq_of(cfs_rq)); } #endif } @@ -3886,7 +3018,7 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) ideal_runtime = sched_slice(cfs_rq, curr); delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; if (delta_exec > ideal_runtime) { - resched_task(rq_of(cfs_rq)->curr); + resched_curr(rq_of(cfs_rq)); /* * The current task ran long enough, ensure it doesn't get * re-elected due to buddy favours. @@ -3910,7 +3042,7 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) return; if (delta > ideal_runtime) - resched_task(rq_of(cfs_rq)->curr); + resched_curr(rq_of(cfs_rq)); } static void @@ -3925,6 +3057,7 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) */ update_stats_wait_end(cfs_rq, se); __dequeue_entity(cfs_rq, se); + update_entity_load_avg(se, 1); } update_stats_curr_start(cfs_rq, se); @@ -3953,17 +3086,36 @@ wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se); * 3) pick the "last" process, for cache locality * 4) do not run the "skip" process, if something else is available */ -static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) +static struct sched_entity * +pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr) { - struct sched_entity *se = __pick_first_entity(cfs_rq); - struct sched_entity *left = se; + struct sched_entity *left = __pick_first_entity(cfs_rq); + struct sched_entity *se; + + /* + * If curr is set we have to see if its left of the leftmost entity + * still in the tree, provided there was anything in the tree at all. + */ + if (!left || (curr && entity_before(curr, left))) + left = curr; + + se = left; /* ideally we run the leftmost entity */ /* * Avoid running the skip buddy, if running something else can * be done without getting too unfair. */ if (cfs_rq->skip == se) { - struct sched_entity *second = __pick_next_entity(se); + struct sched_entity *second; + + if (se == curr) { + second = __pick_first_entity(cfs_rq); + } else { + second = __pick_next_entity(se); + if (!second || (curr && entity_before(curr, second))) + second = curr; + } + if (second && wakeup_preempt_entity(second, left) < 1) se = second; } @@ -3985,7 +3137,7 @@ static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) return se; } -static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq); +static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) { @@ -4031,7 +3183,7 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) * validating it and just reschedule. */ if (queued) { - resched_task(rq_of(cfs_rq)->curr); + resched_curr(rq_of(cfs_rq)); return; } /* @@ -4094,26 +3246,6 @@ static inline u64 sched_cfs_bandwidth_slice(void) return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; } -#ifdef VENDOR_EDIT -static unsigned int count_currnet_cfs_h_tasks(struct cfs_bandwidth *cfs_b) -{ - struct cfs_rq *cfs_rq; - unsigned int task_count = 0; - - rcu_read_lock(); - list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, - throttled_list) { - task_count += cfs_rq->h_nr_running; - } - list_for_each_entry_rcu(cfs_rq, &cfs_b->unthrottled_cfs_rq, - unthrottled_list) { - task_count += cfs_rq->h_nr_running; - } - rcu_read_unlock(); - return task_count?:1; -} -#endif - /* * Replenish runtime according to assigned quota and update expiration time. * We use sched_clock_cpu directly instead of rq->clock to avoid adding @@ -4124,22 +3256,12 @@ static unsigned int count_currnet_cfs_h_tasks(struct cfs_bandwidth *cfs_b) void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) { u64 now; -#ifdef VENDOR_EDIT - unsigned int nr_h_tasks; -#endif if (cfs_b->quota == RUNTIME_INF) return; now = sched_clock_cpu(smp_processor_id()); -#ifdef VENDOR_EDIT - nr_h_tasks = count_currnet_cfs_h_tasks(cfs_b); - if (cfs_b->quota_per_task != 0 && nr_h_tasks != 0) - cfs_b->runtime = min(cfs_b->quota, cfs_b->quota_per_task * nr_h_tasks); - else -#endif - cfs_b->runtime = cfs_b->quota; - + cfs_b->runtime = cfs_b->quota; cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period); } @@ -4154,7 +3276,7 @@ static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq) if (unlikely(cfs_rq->throttle_count)) return cfs_rq->throttled_clock_task; - return rq_of(cfs_rq)->clock_task - cfs_rq->throttled_clock_task_time; + return rq_clock_task(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; } /* returns 0 on failure to allocate runtime */ @@ -4210,10 +3332,9 @@ static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) static void expire_cfs_rq_runtime(struct cfs_rq *cfs_rq) { struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); - struct rq *rq = rq_of(cfs_rq); /* if the deadline is ahead of our clock, nothing to do */ - if (likely((s64)(rq->clock - cfs_rq->runtime_expires) < 0)) + if (likely((s64)(rq_clock(rq_of(cfs_rq)) - cfs_rq->runtime_expires) < 0)) return; if (cfs_rq->runtime_remaining < 0) @@ -4239,11 +3360,7 @@ static void expire_cfs_rq_runtime(struct cfs_rq *cfs_rq) } } -#ifdef VENDOR_EDIT -static atomic_t skip_throttle = ATOMIC_INIT(0); -#endif -static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, - unsigned long delta_exec) +static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) { /* dock delta_exec before expiring quota (as it could span periods) */ cfs_rq->runtime_remaining -= delta_exec; @@ -4256,17 +3373,12 @@ static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, * if we're unable to extend our runtime we resched so that the active * hierarchy can be throttled */ - -#ifdef VENDOR_EDIT - if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr) && !atomic_read(&skip_throttle)) -#else if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) -#endif - resched_task(rq_of(cfs_rq)->curr); + resched_curr(rq_of(cfs_rq)); } static __always_inline -void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, unsigned long delta_exec) +void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) { if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) return; @@ -4279,33 +3391,6 @@ static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) return cfs_bandwidth_used() && cfs_rq->throttled; } -/* - * Check if task is part of a hierarchy where some cfs_rq does not have any - * runtime left. - * - * We can't rely on throttled_hierarchy() to do this test, as - * cfs_rq->throttle_count will not be updated yet when this function is called - * from scheduler_tick() - */ -static int task_will_be_throttled(struct task_struct *p) -{ - struct sched_entity *se = &p->se; - struct cfs_rq *cfs_rq; - - if (!cfs_bandwidth_used()) - return 0; - - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - if (!cfs_rq->runtime_enabled) - continue; - if (cfs_rq->runtime_remaining <= 0) - return 1; - } - - return 0; -} - /* check whether cfs_rq, or any parent, is throttled */ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) { @@ -4339,7 +3424,7 @@ static int tg_unthrottle_up(struct task_group *tg, void *data) #ifdef CONFIG_SMP if (!cfs_rq->throttle_count) { /* adjust cfs_rq_clock_task() */ - cfs_rq->throttled_clock_task_time += rq->clock_task - + cfs_rq->throttled_clock_task_time += rq_clock_task(rq) - cfs_rq->throttled_clock_task; } #endif @@ -4354,16 +3439,12 @@ static int tg_throttle_down(struct task_group *tg, void *data) /* group is entering throttled state, stop time */ if (!cfs_rq->throttle_count) - cfs_rq->throttled_clock_task = rq->clock_task; + cfs_rq->throttled_clock_task = rq_clock_task(rq); cfs_rq->throttle_count++; return 0; } -#ifdef VENDOR_EDIT -static atomic_t throttle_ongoing = ATOMIC_INIT(0); -static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b); -#endif static void throttle_cfs_rq(struct cfs_rq *cfs_rq) { struct rq *rq = rq_of(cfs_rq); @@ -4371,15 +3452,6 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) struct sched_entity *se; long task_delta, dequeue = 1; -#ifdef VENDOR_EDIT - atomic_inc(&throttle_ongoing); - if (atomic_read(&skip_throttle)) { - atomic_dec(&throttle_ongoing); - pr_debug("SMSG: critical tasks are working, skip throttle\n"); - return; - } -#endif - se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; /* freeze hierarchy runnable averages while throttled */ @@ -4397,44 +3469,25 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) if (dequeue) dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP); qcfs_rq->h_nr_running -= task_delta; - dec_throttled_cfs_rq_hmp_stats(&qcfs_rq->hmp_stats, cfs_rq); if (qcfs_rq->load.weight) dequeue = 0; } - if (!se) { - sched_update_nr_prod(cpu_of(rq), task_delta, false); - rq->nr_running -= task_delta; - dec_throttled_cfs_rq_hmp_stats(&rq->hmp_stats, cfs_rq); - } + if (!se) + sub_nr_running(rq, task_delta); cfs_rq->throttled = 1; - cfs_rq->throttled_clock = rq->clock; + cfs_rq->throttled_clock = rq_clock(rq); raw_spin_lock(&cfs_b->lock); /* * Add to the _head_ of the list, so that an already-started * distribute_cfs_runtime will not see us */ list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); -#ifdef VENDOR_EDIT - list_del_rcu(&cfs_rq->unthrottled_list); -#endif if (!cfs_b->timer_active) __start_cfs_bandwidth(cfs_b, false); - if (atomic_read(&skip_throttle)) - start_cfs_slack_bandwidth(cfs_b); raw_spin_unlock(&cfs_b->lock); -#ifdef VENDOR_EDIT - atomic_dec(&throttle_ongoing); -#endif - - /* Log effect on hmp stats after throttling */ - trace_sched_cpu_load(rq, idle_cpu(cpu_of(rq)), - mostly_idle_cpu(cpu_of(rq)), - sched_irqload(cpu_of(rq)), - power_cost_at_freq(cpu_of(rq), 0), - cpu_temp(cpu_of(rq))); } void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) @@ -4444,20 +3497,18 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) struct sched_entity *se; int enqueue = 1; long task_delta; - struct cfs_rq *tcfs_rq = cfs_rq; - se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; + se = cfs_rq->tg->se[cpu_of(rq)]; cfs_rq->throttled = 0; + + update_rq_clock(rq); + raw_spin_lock(&cfs_b->lock); - cfs_b->throttled_time += rq->clock - cfs_rq->throttled_clock; + cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; list_del_rcu(&cfs_rq->throttled_list); -#ifdef VENDOR_EDIT - list_add_rcu(&cfs_rq->unthrottled_list, &cfs_b->unthrottled_cfs_rq); -#endif raw_spin_unlock(&cfs_b->lock); - update_rq_clock(rq); /* update hierarchical throttle state */ walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); @@ -4473,28 +3524,17 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) if (enqueue) enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP); cfs_rq->h_nr_running += task_delta; - inc_throttled_cfs_rq_hmp_stats(&cfs_rq->hmp_stats, tcfs_rq); if (cfs_rq_throttled(cfs_rq)) break; } - if (!se) { - sched_update_nr_prod(cpu_of(rq), task_delta, true); - rq->nr_running += task_delta; - inc_throttled_cfs_rq_hmp_stats(&rq->hmp_stats, tcfs_rq); - } + if (!se) + add_nr_running(rq, task_delta); /* determine whether we need to wake up potentially idle cpu */ if (rq->curr == rq->idle && rq->cfs.nr_running) - resched_task(rq->curr); - - /* Log effect on hmp stats after un-throttling */ - trace_sched_cpu_load(rq, idle_cpu(cpu_of(rq)), - mostly_idle_cpu(cpu_of(rq)), - sched_irqload(cpu_of(rq)), - power_cost_at_freq(cpu_of(rq), 0), - cpu_temp(cpu_of(rq))); + resched_curr(rq); } static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, @@ -4521,10 +3561,6 @@ static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, cfs_rq->runtime_remaining += runtime; cfs_rq->runtime_expires = expires; - /* when skip_throttle, refill little time to unthrottle all cfs_rq */ - if (atomic_read(&skip_throttle) && cfs_rq->runtime_remaining <= 0) - cfs_rq->runtime_remaining = 1; - /* we check whether we're throttled above */ if (cfs_rq->runtime_remaining > 0) unthrottle_cfs_rq(cfs_rq); @@ -4532,8 +3568,7 @@ static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, next: raw_spin_unlock(&rq->lock); - /* when skip_throttle, need to bring throttle cfs_rq back asap */ - if (!remaining && !atomic_read(&skip_throttle)) + if (!remaining) break; } rcu_read_unlock(); @@ -4563,11 +3598,7 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) * idle depends on !throttled (for the case of a large deficit), and if * we're going inactive then everything else can be deferred */ -#ifdef VENDOR_EDIT - if (cfs_b->idle && !throttled && !atomic_read(&skip_throttle)) -#else if (cfs_b->idle && !throttled) -#endif goto out_deactivate; /* @@ -4597,7 +3628,7 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) * in us over-using our runtime if it is all used during this loop, but * only by limited amounts in that extreme case. */ - while (throttled && (cfs_b->runtime > 0 || atomic_read(&skip_throttle))) { + while (throttled && cfs_b->runtime > 0) { runtime = cfs_b->runtime; raw_spin_unlock(&cfs_b->lock); /* we can't nest cfs_b->lock while distributing bandwidth */ @@ -4726,7 +3757,7 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) expires = cfs_b->runtime_expires; raw_spin_unlock(&cfs_b->lock); - if (!runtime && !atomic_read(&skip_throttle)) + if (!runtime) return; runtime = distribute_cfs_runtime(cfs_b, runtime, expires); @@ -4737,38 +3768,6 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) raw_spin_unlock(&cfs_b->lock); } -#ifdef VENDOR_EDIT -void skip_cfs_throttle(int skip) -{ - struct cfs_bandwidth *cfs_b = &task_group(current)->cfs_bandwidth; - if (skip) { - atomic_inc(&skip_throttle); - while (atomic_read(&throttle_ongoing) && cfs_b->quota != RUNTIME_INF) - schedule(); - } else - atomic_dec(&skip_throttle); - - pr_debug_ratelimited("SMSG: current skip_throttle value=%d\n", atomic_read(&skip_throttle)); -} -#endif - -#ifdef VENDOR_EDIT -static inline void enter_cfs_throttling(atomic_t *in_progress) -{ - atomic_inc(in_progress); -} - -static inline void exit_cfs_throttling(atomic_t *in_progress) -{ - atomic_dec(in_progress); -} - -static inline bool is_throttling_ongoing(atomic_t *in_progress) -{ - return atomic_read(in_progress) == 0 ? false:true; -} -#endif - /* * When a group wakes up we want to make sure that its quota is not already * expired/exceeded, otherwise it may be allowed to steal additional ticks of @@ -4776,81 +3775,43 @@ static inline bool is_throttling_ongoing(atomic_t *in_progress) */ static void check_enqueue_throttle(struct cfs_rq *cfs_rq) { -#ifdef VENDOR_EDIT - atomic_t *tip = &cfs_rq->throttling_in_progress; -#endif if (!cfs_bandwidth_used()) return; -#ifdef VENDOR_EDIT - enter_cfs_throttling(tip); -#endif - /* an active group must be handled by the update_curr()->put() path */ - if (!cfs_rq->runtime_enabled || cfs_rq->curr) { -#ifdef VENDOR_EDIT - exit_cfs_throttling(tip); -#endif + if (!cfs_rq->runtime_enabled || cfs_rq->curr) return; - } /* ensure the group is not already throttled */ - if (cfs_rq_throttled(cfs_rq)) { -#ifdef VENDOR_EDIT - exit_cfs_throttling(tip); -#endif + if (cfs_rq_throttled(cfs_rq)) return; - } /* update runtime allocation */ account_cfs_rq_runtime(cfs_rq, 0); if (cfs_rq->runtime_remaining <= 0) throttle_cfs_rq(cfs_rq); -#ifdef VENDOR_EDIT - exit_cfs_throttling(tip); -#endif } /* conditionally throttle active cfs_rq's from put_prev_entity() */ -static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq) +static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { -#ifdef VENDOR_EDIT - atomic_t *tip = &cfs_rq->throttling_in_progress; -#endif if (!cfs_bandwidth_used()) - return; + return false; -#ifdef VENDOR_EDIT - enter_cfs_throttling(tip); -#endif - if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) { -#ifdef VENDOR_EDIT - exit_cfs_throttling(tip); -#endif - return; - } + if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) + return false; /* * it's possible for a throttled entity to be forced into a running * state (e.g. set_curr_task), in this case we're finished. */ - if (cfs_rq_throttled(cfs_rq)) { -#ifdef VENDOR_EDIT - exit_cfs_throttling(tip); -#endif - return; - } + if (cfs_rq_throttled(cfs_rq)) + return true; throttle_cfs_rq(cfs_rq); -#ifdef VENDOR_EDIT - exit_cfs_throttling(tip); -#endif + return true; } -static inline u64 default_cfs_period(void); -static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun); -static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b); - static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) { struct cfs_bandwidth *cfs_b = @@ -4891,9 +3852,6 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) cfs_b->period = ns_to_ktime(default_cfs_period()); INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); -#ifdef VENDOR_EDIT - INIT_LIST_HEAD(&cfs_b->unthrottled_cfs_rq); -#endif hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); cfs_b->period_timer.function = sched_cfs_period_timer; hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); @@ -4903,11 +3861,7 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) { cfs_rq->runtime_enabled = 0; -#ifdef VENDOR_EDIT - INIT_LIST_HEAD(&cfs_rq->unthrottled_list); -#endif INIT_LIST_HEAD(&cfs_rq->throttled_list); - init_cfs_rq_hmp_stats(cfs_rq); } /* requires cfs_b->lock, may release to reprogram timer */ @@ -4943,36 +3897,21 @@ static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) static void __maybe_unused update_runtime_enabled(struct rq *rq) { struct cfs_rq *cfs_rq; -#ifdef VENDOR_EDIT - struct task_group *tg; - rcu_read_lock(); - for_each_cfs_rq(cfs_rq, tg, cpu_of(rq)) { -#else for_each_leaf_cfs_rq(rq, cfs_rq) { -#endif struct cfs_bandwidth *cfs_b = &cfs_rq->tg->cfs_bandwidth; raw_spin_lock(&cfs_b->lock); cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; raw_spin_unlock(&cfs_b->lock); } -#ifdef VENDOR_EDIT - rcu_read_unlock(); -#endif } -static void unthrottle_offline_cfs_rqs(struct rq *rq) +static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) { struct cfs_rq *cfs_rq; -#ifdef VENDOR_EDIT - struct task_group *tg; - rcu_read_lock(); - for_each_cfs_rq(cfs_rq, tg, cpu_of(rq)) { -#else for_each_leaf_cfs_rq(rq, cfs_rq) { -#endif if (!cfs_rq->runtime_enabled) continue; @@ -4986,28 +3925,20 @@ static void unthrottle_offline_cfs_rqs(struct rq *rq) * in take_cpu_down(), so we prevent new cfs throttling here. */ cfs_rq->runtime_enabled = 0; -#ifdef VENDOR_EDIT - while (is_throttling_ongoing(&cfs_rq->throttling_in_progress)) - ; -#endif if (cfs_rq_throttled(cfs_rq)) unthrottle_cfs_rq(cfs_rq); } -#ifdef VENDOR_EDIT - rcu_read_unlock(); -#endif } #else /* CONFIG_CFS_BANDWIDTH */ static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq) { - return rq_of(cfs_rq)->clock_task; + return rq_clock_task(rq_of(cfs_rq)); } -static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, - unsigned long delta_exec) {} -static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} +static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} +static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; } static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} @@ -5055,31 +3986,24 @@ static void hrtick_start_fair(struct rq *rq, struct task_struct *p) WARN_ON(task_rq(p) != rq); - if (rq->cfs.h_nr_running > 1) { + if (cfs_rq->nr_running > 1) { u64 slice = sched_slice(cfs_rq, se); u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; s64 delta = slice - ran; if (delta < 0) { if (rq->curr == p) - resched_task(p); + resched_curr(rq); return; } - - /* - * Don't schedule slices shorter than 10000ns, that just - * doesn't make sense. Rely on vruntime for fairness. - */ - if (rq->curr != p) - delta = max_t(s64, 10000LL, delta); - hrtick_start(rq, delta); } } /* * called from enqueue/dequeue and updates the hrtick when the - * current task is from our class. + * current task is from our class and nr_running is low enough + * to matter. */ static void hrtick_update(struct rq *rq) { @@ -5088,7 +4012,8 @@ static void hrtick_update(struct rq *rq) if (!hrtick_enabled(rq) || curr->sched_class != &fair_sched_class) return; - hrtick_start_fair(rq, curr); + if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency) + hrtick_start_fair(rq, curr); } #else /* !CONFIG_SCHED_HRTICK */ static inline void @@ -5101,6 +4026,13 @@ static inline void hrtick_update(struct rq *rq) } #endif +static unsigned int capacity_margin = 1280; /* ~20% margin */ + +static bool cpu_overutilized(int cpu); +static unsigned long get_cpu_usage(int cpu); +static inline unsigned long get_boosted_cpu_usage(int cpu); +struct static_key __sched_energy_freq __read_mostly = STATIC_KEY_INIT_FALSE; + /* * The enqueue_task method is called before nr_running is * increased. Here we update the fair scheduling stats and @@ -5111,6 +4043,8 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) { struct cfs_rq *cfs_rq; struct sched_entity *se = &p->se; + int task_new = flags & ENQUEUE_WAKEUP_NEW; + int task_wakeup = flags & ENQUEUE_WAKEUP; for_each_sched_entity(se) { if (se->on_rq) @@ -5127,7 +4061,6 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (cfs_rq_throttled(cfs_rq)) break; cfs_rq->h_nr_running++; - inc_cfs_rq_hmp_stats(cfs_rq, p, 1); flags = ENQUEUE_WAKEUP; } @@ -5135,7 +4068,6 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); cfs_rq->h_nr_running++; - inc_cfs_rq_hmp_stats(cfs_rq, p, 1); if (cfs_rq_throttled(cfs_rq)) break; @@ -5146,8 +4078,31 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (!se) { update_rq_runnable_avg(rq, rq->nr_running); - inc_nr_running(rq); - inc_rq_hmp_stats(rq, p, 1); + add_nr_running(rq, 1); + if (!task_new && !rq->rd->overutilized && + cpu_overutilized(rq->cpu)) + rq->rd->overutilized = true; + + schedtune_enqueue_task(p, cpu_of(rq)); + + /* + * We want to trigger a freq switch request only for tasks that + * are waking up; this is because we get here also during + * load balancing, but in these cases it seems wise to trigger + * as single request after load balancing is done. + * + * Also, we add a margin (same ~20% used for the tipping point) + * to our request to provide some head room if p's utilization + * further increases. + */ + if (sched_energy_freq() && (task_new || task_wakeup)) { + unsigned long req_cap = + get_boosted_cpu_usage(cpu_of(rq)); + + req_cap = req_cap * capacity_margin + >> SCHED_CAPACITY_SHIFT; + cpufreq_sched_set_cap(cpu_of(rq), req_cap); + } } hrtick_update(rq); } @@ -5178,7 +4133,6 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (cfs_rq_throttled(cfs_rq)) break; cfs_rq->h_nr_running--; - dec_cfs_rq_hmp_stats(cfs_rq, p, 1); /* Don't dequeue parent if it has other entities besides us */ if (cfs_rq->load.weight) { @@ -5199,7 +4153,6 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); cfs_rq->h_nr_running--; - dec_cfs_rq_hmp_stats(cfs_rq, p, 1); if (cfs_rq_throttled(cfs_rq)) break; @@ -5209,9 +4162,33 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) } if (!se) { - dec_nr_running(rq); + sub_nr_running(rq, 1); update_rq_runnable_avg(rq, 1); - dec_rq_hmp_stats(rq, p, 1); + + schedtune_dequeue_task(p, cpu_of(rq)); + + /* + * We want to trigger a freq switch request only for tasks that + * are going to sleep; this is because we get here also during + * load balancing, but in these cases it seems wise to trigger + * as single request after load balancing is done. + * + * Also, we add a margin (same ~20% used for the tipping point) + * to our request to provide some head room if p's utilization + * further increases. + */ + if (sched_energy_freq() && task_sleep) { + unsigned long req_cap = + get_boosted_cpu_usage(cpu_of(rq)); + + if (rq->cfs.nr_running) { + req_cap = req_cap * capacity_margin + >> SCHED_CAPACITY_SHIFT; + cpufreq_sched_set_cap(cpu_of(rq), req_cap); + } else { + cpufreq_sched_reset_cap(cpu_of(rq)); + } + } } hrtick_update(rq); } @@ -5220,7 +4197,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) /* Used instead of source_load when we know the type == 0 */ static unsigned long weighted_cpuload(const int cpu) { - return cpu_rq(cpu)->load.weight; + return cpu_rq(cpu)->cfs.runnable_load_avg; } /* @@ -5256,22 +4233,45 @@ static unsigned long target_load(int cpu, int type) return max(rq->cpu_load[type-1], total); } -static unsigned long power_of(int cpu) +static unsigned long capacity_of(int cpu) +{ + return cpu_rq(cpu)->cpu_capacity; +} + +unsigned long capacity_orig_of(int cpu) { - return cpu_rq(cpu)->cpu_power; + return cpu_rq(cpu)->cpu_capacity_orig; } static unsigned long cpu_avg_load_per_task(int cpu) { struct rq *rq = cpu_rq(cpu); - unsigned long nr_running = ACCESS_ONCE(rq->nr_running); + unsigned long nr_running = ACCESS_ONCE(rq->cfs.h_nr_running); + unsigned long load_avg = rq->cfs.runnable_load_avg; if (nr_running) - return rq->load.weight / nr_running; + return load_avg / nr_running; return 0; } +static void record_wakee(struct task_struct *p) +{ + /* + * Rough decay (wiping) for cost saving, don't worry + * about the boundary, really active task won't care + * about the loss. + */ + if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { + current->wakee_flips >>= 1; + current->wakee_flip_decay_ts = jiffies; + } + + if (current->last_wakee != p) { + current->last_wakee = p; + current->wakee_flips++; + } +} static void task_waking_fair(struct task_struct *p) { @@ -5292,6 +4292,7 @@ static void task_waking_fair(struct task_struct *p) #endif se->vruntime -= min_vruntime; + record_wakee(p); } #ifdef CONFIG_FAIR_GROUP_SCHED @@ -5375,50 +4376,493 @@ static long effective_load(struct task_group *tg, int cpu, long wl, long wg) else wl = tg->shares; - /* - * Per the above, wl is the new se->load.weight value; since - * those are clipped to [MIN_SHARES, ...) do so now. See - * calc_cfs_shares(). - */ - if (wl < MIN_SHARES) - wl = MIN_SHARES; + /* + * Per the above, wl is the new se->load.weight value; since + * those are clipped to [MIN_SHARES, ...) do so now. See + * calc_cfs_shares(). + */ + if (wl < MIN_SHARES) + wl = MIN_SHARES; + + /* + * wl = dw_i = S * (s'_i - s_i); see (3) + */ + wl -= se->load.weight; + + /* + * Recursively apply this logic to all parent groups to compute + * the final effective load change on the root group. Since + * only the @tg group gets extra weight, all parent groups can + * only redistribute existing shares. @wl is the shift in shares + * resulting from this level per the above. + */ + wg = 0; + } + + return wl; +} + +#else + +static long effective_load(struct task_group *tg, int cpu, long wl, long wg) +{ + return wl; +} + +#endif + +/* + * Returns the current capacity of cpu after applying both + * cpu and freq scaling. + */ +unsigned long capacity_curr_of(int cpu) +{ + return cpu_rq(cpu)->cpu_capacity_orig * + arch_scale_freq_capacity(NULL, cpu) + >> SCHED_CAPACITY_SHIFT; +} + +/* + * get_cpu_usage returns the amount of capacity of a CPU that is used by CFS + * tasks. The unit of the return value must be the one of capacity so we can + * compare the usage with the capacity of the CPU that is available for CFS + * task (ie cpu_capacity). + * + * cfs.utilization_load_avg is the sum of running time of runnable tasks on a + * CPU. It represents the amount of utilization of a CPU in the range + * [0..capacity_orig] where capacity_orig is the cpu_capacity available at the + * highest frequency (arch_scale_freq_capacity()). The usage of a CPU converges + * towards a sum equal to or less than the current capacity (capacity_curr <= + * capacity_orig) of the CPU because it is the running time on this CPU scaled + * by capacity_curr. Nevertheless, cfs.utilization_load_avg can be higher than + * capacity_curr or even higher than capacity_orig because of unfortunate + * rounding in avg_period and running_load_avg or just after migrating tasks + * (and new task wakeups) until the average stabilizes with the new running + * time. We need to check that the usage stays into the range + * [0..capacity_orig] and cap if necessary. Without capping the usage, a group + * could be seen as overloaded (CPU0 usage at 121% + CPU1 usage at 80%) whereas + * CPU1 has 20% of available capacity. We allow usage to overshoot + * capacity_curr (but not capacity_orig) as it useful for predicting the + * capacity required after task migrations (scheduler-driven DVFS). + */ +static unsigned long __get_cpu_usage(int cpu, int delta) +{ + int sum; + unsigned long usage = cpu_rq(cpu)->cfs.utilization_load_avg; + unsigned long blocked = cpu_rq(cpu)->cfs.utilization_blocked_avg; + unsigned long capacity_orig = capacity_orig_of(cpu); + + sum = usage + blocked + delta; + + if (sum < 0) + return 0; + + if (sum >= capacity_orig) + return capacity_orig; + + return sum; +} + +static unsigned long get_cpu_usage(int cpu) +{ + return __get_cpu_usage(cpu, 0); +} + +static inline bool energy_aware(void) +{ + return sched_feat(ENERGY_AWARE); +} + +struct energy_env { + struct sched_group *sg_top; + struct sched_group *sg_cap; + int cap_idx; + int usage_delta; + int src_cpu; + int dst_cpu; + int energy; + int energy_payoff; + struct task_struct *task; + struct { + int before; + int after; + int diff; + int delta; + } nrg; + struct { + int before; + int after; + int delta; + } cap; +}; + +/* + * __cpu_norm_usage() returns the cpu usage relative to a specific capacity, + * i.e. it's busy ratio, in the range [0..SCHED_LOAD_SCALE] which is useful for + * energy calculations. Using the scale-invariant usage returned by + * get_cpu_usage() and approximating scale-invariant usage by: + * + * usage ~ (curr_freq/max_freq)*1024 * capacity_orig/1024 * running_time/time + * + * the normalized usage can be found using the specific capacity. + * + * capacity = capacity_orig * curr_freq/max_freq + * + * norm_usage = running_time/time ~ usage/capacity + */ +static unsigned long __cpu_norm_usage(int cpu, unsigned long capacity, int delta) +{ + int usage = __get_cpu_usage(cpu, delta); + + if (usage >= capacity) + return SCHED_CAPACITY_SCALE; + + return (usage << SCHED_CAPACITY_SHIFT)/capacity; +} + +static int calc_usage_delta(struct energy_env *eenv, int cpu) +{ + if (cpu == eenv->src_cpu) + return -eenv->usage_delta; + if (cpu == eenv->dst_cpu) + return eenv->usage_delta; + return 0; +} + +static +unsigned long group_max_usage(struct energy_env *eenv) +{ + int i, delta; + unsigned long max_usage = 0; + + for_each_cpu(i, sched_group_cpus(eenv->sg_cap)) { + delta = calc_usage_delta(eenv, i); + max_usage = max(max_usage, __get_cpu_usage(i, delta)); + } + + return max_usage; +} + +/* + * group_norm_usage() returns the approximated group usage relative to it's + * current capacity (busy ratio) in the range [0..SCHED_LOAD_SCALE] for use in + * energy calculations. Since task executions may or may not overlap in time in + * the group the true normalized usage is between max(cpu_norm_usage(i)) and + * sum(cpu_norm_usage(i)) when iterating over all cpus in the group, i. The + * latter is used as the estimate as it leads to a more pessimistic energy + * estimate (more busy). + */ +static unsigned +long group_norm_usage(struct energy_env *eenv, struct sched_group *sg) +{ + int i, delta; + unsigned long usage_sum = 0; + unsigned long capacity = sg->sge->cap_states[eenv->cap_idx].cap; + + for_each_cpu(i, sched_group_cpus(sg)) { + delta = calc_usage_delta(eenv, i); + usage_sum += __cpu_norm_usage(i, capacity, delta); + } + + if (usage_sum > SCHED_CAPACITY_SCALE) + return SCHED_CAPACITY_SCALE; + return usage_sum; +} + +static int find_new_capacity(struct energy_env *eenv, + struct sched_group_energy *sge) +{ + int idx; + unsigned long util = group_max_usage(eenv); + + for (idx = 0; idx < sge->nr_cap_states; idx++) { + if (sge->cap_states[idx].cap >= util) + return idx; + } + + eenv->cap_idx = idx; + + return idx; +} + +static bool cpu_overutilized(int cpu) +{ + return (capacity_of(cpu) * 1024) < + (get_cpu_usage(cpu) * capacity_margin); +} + +static int group_idle_state(struct sched_group *sg) +{ + int i, state = INT_MAX; + + /* Find the shallowest idle state in the sched group. */ + for_each_cpu(i, sched_group_cpus(sg)) + state = min(state, idle_get_state_idx(cpu_rq(i))); + + /* Transform system into sched domain idle state. */ + if (sg->sge->nr_idle_states_below > 1) + state -= sg->sge->nr_idle_states_below - 1; + + /* Clamp state to the range of sched domain idle states. */ + return clamp_t(int, state, 0, sg->sge->nr_idle_states - 1); +} + +/* + * sched_group_energy(): Returns absolute energy consumption of cpus belonging + * to the sched_group including shared resources shared only by members of the + * group. Iterates over all cpus in the hierarchy below the sched_group starting + * from the bottom working it's way up before going to the next cpu until all + * cpus are covered at all levels. The current implementation is likely to + * gather the same usage statistics multiple times. This can probably be done in + * a faster but more complex way. + */ +static unsigned int sched_group_energy(struct energy_env *eenv) +{ + struct sched_domain *sd; + int cpu, total_energy = 0; + struct cpumask visit_cpus; + struct sched_group *sg; + + WARN_ON(!eenv->sg_top->sge); + + cpumask_copy(&visit_cpus, sched_group_cpus(eenv->sg_top)); + + while (!cpumask_empty(&visit_cpus)) { + struct sched_group *sg_shared_cap = NULL; + + cpu = cpumask_first(&visit_cpus); + + /* + * Is the group utilization affected by cpus outside this + * sched_group? + */ + sd = highest_flag_domain(cpu, SD_SHARE_CAP_STATES); + if (!sd) + /* + * We most probably raced with hotplug; returning a + * wrong energy estimation is better than entering an + * infinite loop. + */ + break; + + if (sd->parent) + sg_shared_cap = sd->parent->groups; + + for_each_domain(cpu, sd) { + sg = sd->groups; + + /* Has this sched_domain already been visited? */ + if (sd->child && group_first_cpu(sg) != cpu) + break; + + do { + unsigned long group_util; + int sg_busy_energy, sg_idle_energy; + int cap_idx, idle_idx; + + if (sg_shared_cap && sg_shared_cap->group_weight >= sg->group_weight) + eenv->sg_cap = sg_shared_cap; + else + eenv->sg_cap = sg; + + cap_idx = find_new_capacity(eenv, sg->sge); + + if (sg->group_weight == 1) { + /* Remove capacity of src CPU (before task move) */ + if (eenv->usage_delta == 0 && + cpumask_test_cpu(eenv->src_cpu, sched_group_cpus(sg))) { + eenv->cap.before = sg->sge->cap_states[cap_idx].cap; + eenv->cap.delta -= eenv->cap.before; + } + /* Add capacity of dst CPU (after task move) */ + if (eenv->usage_delta != 0 && + cpumask_test_cpu(eenv->dst_cpu, sched_group_cpus(sg))) { + eenv->cap.after = sg->sge->cap_states[cap_idx].cap; + eenv->cap.delta += eenv->cap.after; + } + } + + idle_idx = group_idle_state(sg); + group_util = group_norm_usage(eenv, sg); + sg_busy_energy = (group_util * sg->sge->cap_states[cap_idx].power) + >> SCHED_CAPACITY_SHIFT; + sg_idle_energy = ((SCHED_LOAD_SCALE-group_util) + * sg->sge->idle_states[idle_idx].power) + >> SCHED_CAPACITY_SHIFT; + + total_energy += sg_busy_energy + sg_idle_energy; + + if (!sd->child) + cpumask_xor(&visit_cpus, &visit_cpus, sched_group_cpus(sg)); + + if (cpumask_equal(sched_group_cpus(sg), sched_group_cpus(eenv->sg_top))) + goto next_cpu; + + } while (sg = sg->next, sg != sd->groups); + } +next_cpu: + continue; + } + + eenv->energy = total_energy; + return total_energy; +} + +#ifdef CONFIG_SCHED_TUNE +static int energy_diff_evaluate(struct energy_env *eenv) +{ + unsigned int boost; + int nrg_delta; + + /* Return energy diff when boost margin is 0 */ +#ifdef CONFIG_CGROUP_SCHEDTUNE + boost = schedtune_taskgroup_boost(eenv->task); +#else + boost = get_sysctl_sched_cfs_boost(); +#endif + if (boost == 0) + return eenv->nrg.diff; + + /* Compute normalized energy diff */ + nrg_delta = schedtune_normalize_energy(eenv->nrg.diff); + eenv->nrg.delta = nrg_delta; + +#ifdef CONFIG_CGROUP_SCHEDTUNE + eenv->energy_payoff = schedtune_accept_deltas( + eenv->nrg.delta, + eenv->cap.delta, + eenv->task); +#else + eenv->energy_payoff = schedtune_accept_deltas( + eenv->nrg.delta, + eenv->cap.delta); +#endif + + /* + * When SchedTune is enabled, the energy_diff() function will return + * the computed energy payoff value. Since the energy_diff() return + * value is expected to be negative by its callers, this evaluation + * function return a negative value each time the evaluation return a + * positive energy payoff, which is the condition for the acceptance of + * a scheduling decision + */ + return -eenv->energy_payoff; +} +#else /* CONFIG_SCHED_TUNE */ +#define energy_diff_evaluate(eenv) eenv->nrg.diff +#endif + +/* + * energy_diff(): Estimate the energy impact of changing the utilization + * distribution. eenv specifies the change: utilisation amount, source, and + * destination cpu. Source or destination cpu may be -1 in which case the + * utilization is removed from or added to the system (e.g. task wake-up). If + * both are specified, the utilization is migrated. + */ +static int energy_diff(struct energy_env *eenv) +{ + struct sched_domain *sd; + struct sched_group *sg; + int sd_cpu = -1, energy_before = 0, energy_after = 0; + int result; + + struct energy_env eenv_before = { + .usage_delta = 0, + .src_cpu = eenv->src_cpu, + .dst_cpu = eenv->dst_cpu, + .nrg = { 0, 0, 0, 0 }, + .cap = { 0, 0, 0 }, + }; + + if (eenv->src_cpu == eenv->dst_cpu) + return 0; + + sd_cpu = (eenv->src_cpu != -1) ? eenv->src_cpu : eenv->dst_cpu; + sd = rcu_dereference(per_cpu(sd_ea, sd_cpu)); + + if (!sd) + return 0; /* Error */ + + sg = sd->groups; + do { + if (eenv->src_cpu != -1 && cpumask_test_cpu(eenv->src_cpu, + sched_group_cpus(sg))) { + eenv_before.sg_top = eenv->sg_top = sg; + energy_before += sched_group_energy(&eenv_before); + + /* Keep track of SRC cpu (before) capacity */ + eenv->cap.before = eenv_before.cap.before; + eenv->cap.delta = eenv_before.cap.delta; + + energy_after += sched_group_energy(eenv); + + /* src_cpu and dst_cpu may belong to the same group */ + continue; + } + + if (eenv->dst_cpu != -1 && cpumask_test_cpu(eenv->dst_cpu, + sched_group_cpus(sg))) { + eenv_before.sg_top = eenv->sg_top = sg; + energy_before += sched_group_energy(&eenv_before); + energy_after += sched_group_energy(eenv); + } + } while (sg = sg->next, sg != sd->groups); + + eenv->nrg.before = energy_before; + eenv->nrg.after = energy_after; + eenv->nrg.diff = eenv->nrg.after - eenv->nrg.before; + eenv->energy_payoff = 0; + + result = energy_diff_evaluate(eenv); +#if 0 + // TJK: fix tracint + trace_sched_energy_diff(eenv->task, + eenv->src_cpu, eenv->dst_cpu, eenv->usage_delta, + eenv->nrg.before, eenv->nrg.after, eenv->nrg.diff, + eenv->cap.before, eenv->cap.after, eenv->cap.delta, + eenv->nrg.delta, eenv->energy_payoff); +#endif + return result; +} - /* - * wl = dw_i = S * (s'_i - s_i); see (3) - */ - wl -= se->load.weight; +static int wake_wide(struct task_struct *p) +{ + int factor = this_cpu_read(sd_llc_size); + /* + * Yeah, it's the switching-frequency, could means many wakee or + * rapidly switch, use factor here will just help to automatically + * adjust the loose-degree, so bigger node will lead to more pull. + */ + if (p->wakee_flips > factor) { /* - * Recursively apply this logic to all parent groups to compute - * the final effective load change on the root group. Since - * only the @tg group gets extra weight, all parent groups can - * only redistribute existing shares. @wl is the shift in shares - * resulting from this level per the above. + * wakee is somewhat hot, it needs certain amount of cpu + * resource, so if waker is far more hot, prefer to leave + * it alone. */ - wg = 0; + if (current->wakee_flips > (factor * p->wakee_flips)) + return 1; } - return wl; -} -#else - -static inline unsigned long effective_load(struct task_group *tg, int cpu, - unsigned long wl, unsigned long wg) -{ - return wl; + return 0; } -#endif - static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync) { s64 this_load, load; + s64 this_eff_load, prev_eff_load; int idx, this_cpu, prev_cpu; - unsigned long tl_per_task; struct task_group *tg; unsigned long weight; int balanced; + /* + * If we wake multiple tasks be careful to not bounce + * ourselves around too much. + */ + if (wake_wide(p)) + return 0; + idx = sd->wake_idx; this_cpu = smp_processor_id(); prev_cpu = task_cpu(p); @@ -5450,63 +4894,227 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync) * Otherwise check if either cpus are near enough in load to allow this * task to be woken on this_cpu. */ - if (this_load > 0) { - s64 this_eff_load, prev_eff_load; + this_eff_load = 100; + this_eff_load *= capacity_of(prev_cpu); - this_eff_load = 100; - this_eff_load *= power_of(prev_cpu); + prev_eff_load = 100 + (sd->imbalance_pct - 100) / 2; + prev_eff_load *= capacity_of(this_cpu); + + if (this_load > 0) { this_eff_load *= this_load + effective_load(tg, this_cpu, weight, weight); - prev_eff_load = 100 + (sd->imbalance_pct - 100) / 2; - prev_eff_load *= power_of(this_cpu); prev_eff_load *= load + effective_load(tg, prev_cpu, 0, weight); + } - balanced = this_eff_load <= prev_eff_load; - } else - balanced = true; + balanced = this_eff_load <= prev_eff_load; + + schedstat_inc(p, se.statistics.nr_wakeups_affine_attempts); + + if (!balanced) + return 0; + + schedstat_inc(sd, ttwu_move_affine); + schedstat_inc(p, se.statistics.nr_wakeups_affine); + + return 1; +} + +static inline unsigned long task_utilization(struct task_struct *p) +{ + return p->se.avg.utilization_avg_contrib; +} + +#ifdef CONFIG_SCHED_TUNE + +static unsigned long +schedtune_margin(unsigned long signal, unsigned long boost) +{ + unsigned long long margin = 0; /* - * If the currently running task will sleep within - * a reasonable amount of time then attract this newly - * woken task: + * Signal proportional compensation (SPC) + * + * The Boost (B) value is used to compute a Maring (M) which is + * proportional to the complement of the original Signal (S): + * M = B * (1024-S) + * The obtained M could be used by the caller to "boost" S. */ - if (sync && balanced) - return 1; + margin = SCHED_LOAD_SCALE - signal; + margin *= boost; - schedstat_inc(p, se.statistics.nr_wakeups_affine_attempts); - tl_per_task = cpu_avg_load_per_task(this_cpu); + /* + * Fast integer division by constant: + * Constant : (C) = 100 + * Precision : 0.1% (P) = 0.1 + * Reference : C * 100 / P (R) = 100000 + * + * Thus: + * Shift bifs : ceil(log(R,2)) (S) = 17 + * Mult const : round(2^S/C) (M) = 1311 + * + * + * */ + margin *= 1311; + margin >>= 17; - if (balanced || - (this_load <= load && - this_load + target_load(prev_cpu, idx) <= tl_per_task)) { - /* - * This domain has SD_WAKE_AFFINE and - * p is cache cold in this domain, and - * there is no bad imbalance. - */ - schedstat_inc(sd, ttwu_move_affine); - schedstat_inc(p, se.statistics.nr_wakeups_affine); + return margin; - return 1; +} + +static unsigned long +schedtune_task_margin(struct task_struct *task) +{ + unsigned int boost; + unsigned long utilization; + unsigned long margin; + +#ifdef CONFIG_CGROUP_SCHEDTUNE + boost = schedtune_taskgroup_boost(task); +#else + boost = get_sysctl_sched_cfs_boost(); +#endif + if (boost == 0) + return 0; + + utilization = task_utilization(task); + margin = schedtune_margin(utilization, boost); + + return margin; +} + +static unsigned long +boosted_task_utilization(struct task_struct *task) +{ + unsigned long utilization; + unsigned long margin = 0; + + utilization = task_utilization(task); + + /* + * Boosting of task utilization is enabled only when the scheduler is + * working in energy-aware mode. + */ + if (!task_rq(task)->rd->overutilized) + margin = schedtune_task_margin(task); + + //TJK: fix tracing: trace_sched_boost_task(task, utilization, margin); + + utilization += margin; + + return utilization; +} + +#else /* CONFIG_SCHED_TUNE */ + +static unsigned long +boosted_task_utilization(struct task_struct *task) +{ + return task_utilization(task); +} + +#endif /* CONFIG_SCHED_TUNE */ + +static inline bool __task_fits(struct task_struct *p, int cpu, int usage) +{ + unsigned long capacity = capacity_of(cpu); + + usage += boosted_task_utilization(p); + + return (capacity * 1024) > (usage * capacity_margin); +} + +static inline bool task_fits_capacity(struct task_struct *p, int cpu) +{ + unsigned long capacity = capacity_of(cpu); + unsigned long max_capacity = cpu_rq(cpu)->rd->max_cpu_capacity; + + if (capacity == max_capacity) + return true; + + if (capacity * capacity_margin > max_capacity * 1024) + return true; + + return __task_fits(p, cpu, 0); +} + +static inline bool task_fits_cpu(struct task_struct *p, int cpu) +{ + return __task_fits(p, cpu, get_cpu_usage(cpu)); +} + +#ifdef CONFIG_SCHED_TUNE + +static inline unsigned int +schedtune_cpu_margin(int cpu, unsigned long usage) +{ + unsigned int boost; + unsigned long margin; + +#ifdef CONFIG_CGROUP_SCHEDTUNE + if (usage) + boost = schedtune_cpu_boost(cpu); + else { + boost = 0; + schedtune_idle(cpu); } +#else + boost = get_sysctl_sched_cfs_boost(); +#endif + if (boost == 0) + return 0; + + margin = schedtune_margin(usage, boost); + + return margin; +} + +#else /* CONFIG_SCHED_TUNE */ + +static inline unsigned int +schedtune_cpu_margin(int cpu, unsigned long usage) +{ return 0; } +#endif /* CONFIG_SCHED_TUNE */ + +static inline unsigned long +get_boosted_cpu_usage(int cpu) +{ + unsigned long usage; + unsigned long margin; + + usage = get_cpu_usage(cpu); + margin = schedtune_cpu_margin(cpu, usage); + + // TJK: fix tracing trace_sched_boost_cpu(cpu, usage, margin); + + usage += margin; + return usage; +} + /* * find_idlest_group finds and returns the least busy CPU group within the * domain. */ static struct sched_group * find_idlest_group(struct sched_domain *sd, struct task_struct *p, - int this_cpu, int load_idx) + int this_cpu, int sd_flag) { struct sched_group *idlest = NULL, *group = sd->groups; + struct sched_group *fit_group = NULL, *spare_group = NULL; unsigned long min_load = ULONG_MAX, this_load = 0; + unsigned long fit_capacity = ULONG_MAX; + unsigned long max_spare_capacity = capacity_margin - SCHED_LOAD_SCALE; + int load_idx = sd->forkexec_idx; int imbalance = 100 + (sd->imbalance_pct-100)/2; + if (sd_flag & SD_BALANCE_WAKE) + load_idx = sd->wake_idx; + do { - unsigned long load, avg_load; + unsigned long load, avg_load, spare_capacity; int local_group; int i; @@ -5529,10 +5137,30 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, load = target_load(i, load_idx); avg_load += load; + + /* + * Look for most energy-efficient group that can fit + * that can fit the task. + */ + if (energy_aware() && capacity_of(i) < fit_capacity && + task_fits_cpu(p, i)) { + fit_capacity = capacity_of(i); + fit_group = group; + } + + /* + * Look for group which has most spare capacity on a + * single cpu. + */ + spare_capacity = capacity_of(i) - get_cpu_usage(i); + if (spare_capacity > max_spare_capacity) { + max_spare_capacity = spare_capacity; + spare_group = group; + } } - /* Adjust by relative CPU power of the group */ - avg_load = (avg_load * SCHED_CAPACITY_SCALE) / group->sgp->power; + /* Adjust by relative CPU capacity of the group */ + avg_load = (avg_load * SCHED_CAPACITY_SCALE) / group->sgc->capacity; if (local_group) { this_load = avg_load; @@ -5542,6 +5170,12 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, } } while (group = group->next, group != sd->groups); + if (fit_group) + return fit_group; + + if (spare_group) + return spare_group; + if (!idlest || 100*this_load < imbalance*min_load) return NULL; return idlest; @@ -5554,20 +5188,54 @@ static int find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) { unsigned long load, min_load = ULONG_MAX; - int idlest = -1; + unsigned int min_exit_latency = UINT_MAX; + u64 latest_idle_timestamp = 0; + int least_loaded_cpu = this_cpu; + int shallowest_idle_cpu = -1; int i; /* Traverse only the allowed CPUs */ for_each_cpu_and(i, sched_group_cpus(group), tsk_cpus_allowed(p)) { - load = weighted_cpuload(i); - - if (load < min_load || (load == min_load && i == this_cpu)) { - min_load = load; - idlest = i; + if (task_fits_cpu(p, i)) { + struct rq *rq = cpu_rq(i); + struct cpuidle_state *idle = idle_get_state(rq); + if (idle && idle->exit_latency < min_exit_latency) { + /* + * We give priority to a CPU whose idle state + * has the smallest exit latency irrespective + * of any idle timestamp. + */ + min_exit_latency = idle->exit_latency; + latest_idle_timestamp = rq->idle_stamp; + shallowest_idle_cpu = i; + } else if (idle_cpu(i) && + (!idle || idle->exit_latency == min_exit_latency) && + rq->idle_stamp > latest_idle_timestamp) { + /* + * If equal or no active idle state, then + * the most recently idled CPU might have + * a warmer cache. + */ + latest_idle_timestamp = rq->idle_stamp; + shallowest_idle_cpu = i; + } else if (shallowest_idle_cpu == -1) { + /* + * If we haven't found an idle CPU yet + * pick a non-idle one that can fit the task as + * fallback. + */ + shallowest_idle_cpu = i; + } + } else { + load = weighted_cpuload(i); + if (load < min_load || (load == min_load && i == this_cpu)) { + min_load = load; + least_loaded_cpu = i; + } } } - return idlest; + return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; } /* @@ -5588,11 +5256,6 @@ static int select_idle_sibling(struct task_struct *p, int target) if (i != target && cpus_share_cache(i, target) && idle_cpu(i)) return i; - if (!sysctl_sched_wake_to_idle && - !(current->flags & PF_WAKE_UP_IDLE) && - !(p->flags & PF_WAKE_UP_IDLE)) - return target; - /* * Otherwise, iterate the domains and find an elegible idle cpu. */ @@ -5620,38 +5283,118 @@ static int select_idle_sibling(struct task_struct *p, int target) return target; } +static int energy_aware_wake_cpu(struct task_struct *p, int target) +{ + struct sched_domain *sd; + struct sched_group *sg, *sg_target; + int target_max_cap = INT_MAX; + int target_cpu = task_cpu(p); + int i; + + sd = rcu_dereference(per_cpu(sd_ea, task_cpu(p))); + + if (!sd) + return target; + + sg = sd->groups; + sg_target = sg; + + /* + * Find group with sufficient capacity. We only get here if no cpu is + * overutilized. We may end up overutilizing a cpu by adding the task, + * but that should not be any worse than select_idle_sibling(). + * load_balance() should sort it out later as we get above the tipping + * point. + */ + do { + /* Assuming all cpus are the same in group */ + int max_cap_cpu = group_first_cpu(sg); + + /* + * Assume smaller max capacity means more energy-efficient. + * Ideally we should query the energy model for the right + * answer but it easily ends up in an exhaustive search. + */ + if (capacity_of(max_cap_cpu) < target_max_cap && + task_fits_capacity(p, max_cap_cpu)) { + sg_target = sg; + target_max_cap = capacity_of(max_cap_cpu); + } + } while (sg = sg->next, sg != sd->groups); + + /* Find cpu with sufficient capacity */ + for_each_cpu_and(i, tsk_cpus_allowed(p), sched_group_cpus(sg_target)) { + /* + * p's blocked utilization is still accounted for on prev_cpu + * so prev_cpu will receive a negative bias due the double + * accouting. However, the blocked utilization may be zero. + */ + int new_usage = get_cpu_usage(i) + boosted_task_utilization(p); + + if (new_usage > capacity_orig_of(i)) + continue; + + if (new_usage < capacity_curr_of(i)) { + target_cpu = i; + if (cpu_rq(i)->nr_running) + break; + } + + /* cpu has capacity at higher OPP, keep it as fallback */ + if (target_cpu == task_cpu(p)) + target_cpu = i; + } + + if (target_cpu != task_cpu(p)) { + struct energy_env eenv = { + .usage_delta = task_utilization(p), + .src_cpu = task_cpu(p), + .dst_cpu = target_cpu, + .task = p, + }; + + /* Not enough spare capacity on previous cpu */ + if (cpu_overutilized(task_cpu(p))) + return target_cpu; + + if (energy_diff(&eenv) >= 0) + return task_cpu(p); + } + + return target_cpu; +} + /* - * sched_balance_self: balance the current task (running on cpu) in domains - * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and - * SD_BALANCE_EXEC. + * select_task_rq_fair: Select target runqueue for the waking task in domains + * that have the 'sd_flag' flag set. In practice, this is SD_BALANCE_WAKE, + * SD_BALANCE_FORK, or SD_BALANCE_EXEC. * - * Balance, ie. select the least loaded group. + * Balances load by selecting the idlest cpu in the idlest group, or under + * certain conditions an idle sibling cpu if the domain has SD_WAKE_AFFINE set. * - * Returns the target CPU number, or the same CPU if no balancing is needed. + * Returns the target cpu number. * * preempt must be disabled. */ static int -select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags) +select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags) { struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL; int cpu = smp_processor_id(); - int prev_cpu = task_cpu(p); int new_cpu = cpu; int want_affine = 0; + int want_sibling = true; int sync = wake_flags & WF_SYNC; if (p->nr_cpus_allowed == 1) return prev_cpu; - if (sched_enable_hmp) - return select_best_cpu(p, prev_cpu, 0, sync); + /* Check if prev_cpu can fit us ignoring its current usage */ + if (energy_aware() && !task_fits_capacity(p, prev_cpu)) + want_sibling = false; - if (sd_flag & SD_BALANCE_WAKE) { - if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) - want_affine = 1; - new_cpu = prev_cpu; - } + if (sd_flag & SD_BALANCE_WAKE && want_sibling) + want_affine = cpumask_test_cpu(cpu, tsk_cpus_allowed(p)); rcu_read_lock(); for_each_domain(cpu, tmp) { @@ -5672,16 +5415,18 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags) sd = tmp; } - if (affine_sd) { - if (cpu != prev_cpu && wake_affine(affine_sd, p, sync)) - prev_cpu = cpu; + if (affine_sd && cpu != prev_cpu && wake_affine(affine_sd, p, sync)) + prev_cpu = cpu; - new_cpu = select_idle_sibling(p, prev_cpu); + if (sd_flag & SD_BALANCE_WAKE && want_sibling) { + if (energy_aware() && !cpu_rq(cpu)->rd->overutilized) + new_cpu = energy_aware_wake_cpu(p, prev_cpu); + else + new_cpu = select_idle_sibling(p, prev_cpu); goto unlock; } while (sd) { - int load_idx = sd->forkexec_idx; struct sched_group *group; int weight; @@ -5690,10 +5435,7 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags) continue; } - if (sd_flag & SD_BALANCE_WAKE) - load_idx = sd->wake_idx; - - group = find_idlest_group(sd, p, cpu, load_idx); + group = find_idlest_group(sd, p, cpu, sd_flag); if (!group) { sd = sd->child; continue; @@ -5724,12 +5466,6 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags) return new_cpu; } -/* - * Load-tracking only depends on SMP, FAIR_GROUP_SCHED dependency below may be - * removed when useful for applications beyond shares distribution (e.g. - * load-balance). - */ -#ifdef CONFIG_FAIR_GROUP_SCHED /* * Called immediately before a task is migrated to a new cpu; task_cpu(p) and * cfs_rq_of(p) references at time of call are still valid and identify the @@ -5750,10 +5486,15 @@ migrate_task_rq_fair(struct task_struct *p, int next_cpu) */ if (se->avg.decay_count) { se->avg.decay_count = -__synchronize_entity_decay(se); - atomic64_add(se->avg.load_avg_contrib, &cfs_rq->removed_load); + atomic_long_add(se->avg.load_avg_contrib, + &cfs_rq->removed_load); + atomic_long_add(se->avg.utilization_avg_contrib, + &cfs_rq->removed_utilization); } + + /* We have migrated, no longer consider this task hot */ + se->exec_start = 0; } -#endif #endif /* CONFIG_SMP */ static unsigned long @@ -5845,7 +5586,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ return; /* - * This is possible from callers such as move_task(), in which we + * This is possible from callers such as attach_tasks(), in which we * unconditionally check_prempt_curr() after an enqueue (which may have * lead to a throttle). This both saves work and prevents false * next-buddy nomination below. @@ -5899,7 +5640,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ return; preempt: - resched_task(curr); + resched_curr(rq); /* * Only set the backward buddy when the current task is still * on the rq. This can happen when a wakeup gets interleaved @@ -5916,26 +5657,130 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ set_last_buddy(se); } -static struct task_struct *pick_next_task_fair(struct rq *rq) +static struct task_struct * +pick_next_task_fair(struct rq *rq, struct task_struct *prev) { - struct task_struct *p; struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *se; + struct task_struct *p; + int new_tasks; +again: +#ifdef CONFIG_FAIR_GROUP_SCHED if (!cfs_rq->nr_running) - return NULL; + goto idle; + + if (prev->sched_class != &fair_sched_class) + goto simple; + + /* + * Because of the set_next_buddy() in dequeue_task_fair() it is rather + * likely that a next task is from the same cgroup as the current. + * + * Therefore attempt to avoid putting and setting the entire cgroup + * hierarchy, only change the part that actually changes. + */ + + do { + struct sched_entity *curr = cfs_rq->curr; + + /* + * Since we got here without doing put_prev_entity() we also + * have to consider cfs_rq->curr. If it is still a runnable + * entity, update_curr() will update its vruntime, otherwise + * forget we've ever seen it. + */ + if (curr && curr->on_rq) + update_curr(cfs_rq); + else + curr = NULL; + + /* + * This call to check_cfs_rq_runtime() will do the throttle and + * dequeue its entity in the parent(s). Therefore the 'simple' + * nr_running test will indeed be correct. + */ + if (unlikely(check_cfs_rq_runtime(cfs_rq))) + goto simple; + + se = pick_next_entity(cfs_rq, curr); + cfs_rq = group_cfs_rq(se); + } while (cfs_rq); + + p = task_of(se); + + /* + * Since we haven't yet done put_prev_entity and if the selected task + * is a different task than we started out with, try and touch the + * least amount of cfs_rqs. + */ + if (prev != p) { + struct sched_entity *pse = &prev->se; + + while (!(cfs_rq = is_same_group(se, pse))) { + int se_depth = se->depth; + int pse_depth = pse->depth; + + if (se_depth <= pse_depth) { + put_prev_entity(cfs_rq_of(pse), pse); + pse = parent_entity(pse); + } + if (se_depth >= pse_depth) { + set_next_entity(cfs_rq_of(se), se); + se = parent_entity(se); + } + } + + put_prev_entity(cfs_rq, pse); + set_next_entity(cfs_rq, se); + } + + if (hrtick_enabled(rq)) + hrtick_start_fair(rq, p); + + rq->misfit_task = !task_fits_capacity(p, rq->cpu); + + return p; +simple: + cfs_rq = &rq->cfs; +#endif + + if (!cfs_rq->nr_running) + goto idle; + + put_prev_task(rq, prev); do { - se = pick_next_entity(cfs_rq); + se = pick_next_entity(cfs_rq, NULL); set_next_entity(cfs_rq, se); cfs_rq = group_cfs_rq(se); } while (cfs_rq); p = task_of(se); + if (hrtick_enabled(rq)) hrtick_start_fair(rq, p); + rq->misfit_task = !task_fits_capacity(p, rq->cpu); + return p; + +idle: + rq->misfit_task = 0; + + new_tasks = idle_balance(rq); + /* + * Because idle_balance() releases (and re-acquires) rq->lock, it is + * possible for any higher priority task to appear. In that case we + * must re-start the pick_next_entity() loop. + */ + if (new_tasks < 0) + return RETRY_TASK; + + if (new_tasks > 0) + goto again; + + return NULL; } /* @@ -6029,14 +5874,14 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preemp * * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) * - * P_i is the cpu power (or compute capacity) of cpu i, typically it is the + * C_i is the compute capacity of cpu i, typically it is the * fraction of 'recent' time available for SCHED_OTHER task execution. But it * can also include other factors [XXX]. * * To achieve this balance we define a measure of imbalance which follows * directly from (1): * - * imb_i,j = max{ avg(W/P), W_i/P_i } - min{ avg(W/P), W_j/P_j } (4) + * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4) * * We them move tasks around to minimize the imbalance. In the continuous * function space it is obvious this converges, in the discrete case we get @@ -6123,19 +5968,21 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preemp * rewrite all of this once again.] */ -static unsigned long __read_mostly max_load_balance_interval = HZ/10; +static unsigned long __read_mostly max_load_balance_interval = HZ/10; + +enum fbq_type { regular, remote, all }; + +enum group_type { + group_other = 0, + group_misfit_task, + group_imbalanced, + group_overloaded, +}; #define LBF_ALL_PINNED 0x01 #define LBF_NEED_BREAK 0x02 -#define LBF_SOME_PINNED 0x04 -#define LBF_IGNORE_SMALL_TASKS 0x08 -#define LBF_EA_ACTIVE_BALANCE 0x10 -#define LBF_SCHED_BOOST_ACTIVE_BALANCE 0x20 -#define LBF_BIG_TASK_ACTIVE_BALANCE 0x40 -#define LBF_HMP_ACTIVE_BALANCE (LBF_EA_ACTIVE_BALANCE | \ - LBF_SCHED_BOOST_ACTIVE_BALANCE | \ - LBF_BIG_TASK_ACTIVE_BALANCE) -#define LBF_IGNORE_BIG_TASKS 0x80 +#define LBF_DST_PINNED 0x04 +#define LBF_SOME_PINNED 0x08 struct lb_env { struct sched_domain *sd; @@ -6150,43 +5997,30 @@ struct lb_env { int new_dst_cpu; enum cpu_idle_type idle; long imbalance; + unsigned int src_grp_nr_running; /* The set of CPUs under consideration for load-balancing */ struct cpumask *cpus; - unsigned int busiest_grp_capacity; - unsigned int busiest_nr_running; unsigned int flags; unsigned int loop; unsigned int loop_break; unsigned int loop_max; -}; - -static DEFINE_PER_CPU(bool, dbs_boost_needed); -static DEFINE_PER_CPU(int, dbs_boost_load_moved); -/* - * move_task - move a task from one runqueue to another runqueue. - * Both runqueues must be locked. - */ -static void move_task(struct task_struct *p, struct lb_env *env) -{ - deactivate_task(env->src_rq, p, 0); - set_task_cpu(p, env->dst_cpu); - activate_task(env->dst_rq, p, 0); - check_preempt_curr(env->dst_rq, p, 0); - if (task_notify_on_migrate(p)) - per_cpu(dbs_boost_needed, env->dst_cpu) = true; -} + enum fbq_type fbq_type; + enum group_type busiest_group_type; + struct list_head tasks; +}; /* * Is this task likely cache-hot: */ -static int -task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) +static int task_hot(struct task_struct *p, struct lb_env *env) { s64 delta; + lockdep_assert_held(&env->src_rq->lock); + if (p->sched_class != &fair_sched_class) return 0; @@ -6196,7 +6030,7 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) /* * Buddy candidates are cache hot: */ - if (sched_feat(CACHE_HOT_BUDDY) && this_rq()->nr_running && + if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && (&p->se == cfs_rq_of(&p->se)->next || &p->se == cfs_rq_of(&p->se)->last)) return 1; @@ -6206,11 +6040,98 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) if (sysctl_sched_migration_cost == 0) return 0; - delta = now - p->se.exec_start; + delta = rq_clock_task(env->src_rq) - p->se.exec_start; return delta < (s64)sysctl_sched_migration_cost; } +#ifdef CONFIG_NUMA_BALANCING +/* Returns true if the destination node has incurred more faults */ +static bool migrate_improves_locality(struct task_struct *p, struct lb_env *env) +{ + struct numa_group *numa_group = rcu_dereference(p->numa_group); + int src_nid, dst_nid; + + if (!sched_feat(NUMA_FAVOUR_HIGHER) || !p->numa_faults_memory || + !(env->sd->flags & SD_NUMA)) { + return false; + } + + src_nid = cpu_to_node(env->src_cpu); + dst_nid = cpu_to_node(env->dst_cpu); + + if (src_nid == dst_nid) + return false; + + if (numa_group) { + /* Task is already in the group's interleave set. */ + if (node_isset(src_nid, numa_group->active_nodes)) + return false; + + /* Task is moving into the group's interleave set. */ + if (node_isset(dst_nid, numa_group->active_nodes)) + return true; + + return group_faults(p, dst_nid) > group_faults(p, src_nid); + } + + /* Encourage migration to the preferred node. */ + if (dst_nid == p->numa_preferred_nid) + return true; + + return task_faults(p, dst_nid) > task_faults(p, src_nid); +} + +static bool migrate_degrades_locality(struct task_struct *p, struct lb_env *env) +{ + struct numa_group *numa_group = rcu_dereference(p->numa_group); + int src_nid, dst_nid; + + if (!sched_feat(NUMA) || !sched_feat(NUMA_RESIST_LOWER)) + return false; + + if (!p->numa_faults_memory || !(env->sd->flags & SD_NUMA)) + return false; + + src_nid = cpu_to_node(env->src_cpu); + dst_nid = cpu_to_node(env->dst_cpu); + + if (src_nid == dst_nid) + return false; + + if (numa_group) { + /* Task is moving within/into the group's interleave set. */ + if (node_isset(dst_nid, numa_group->active_nodes)) + return false; + + /* Task is moving out of the group's interleave set. */ + if (node_isset(src_nid, numa_group->active_nodes)) + return true; + + return group_faults(p, dst_nid) < group_faults(p, src_nid); + } + + /* Migrating away from the preferred node is always bad. */ + if (src_nid == p->numa_preferred_nid) + return true; + + return task_faults(p, dst_nid) < task_faults(p, src_nid); +} + +#else +static inline bool migrate_improves_locality(struct task_struct *p, + struct lb_env *env) +{ + return false; +} + +static inline bool migrate_degrades_locality(struct task_struct *p, + struct lb_env *env) +{ + return false; +} +#endif + /* * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? */ @@ -6218,7 +6139,9 @@ static int can_migrate_task(struct task_struct *p, struct lb_env *env) { int tsk_cache_hot = 0; - int twf; + + lockdep_assert_held(&env->src_rq->lock); + /* * We do not migrate tasks that are: * 1) throttled_lb_pair, or @@ -6229,37 +6152,13 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu)) return 0; - if (nr_big_tasks(env->src_rq) && - capacity(env->dst_rq) > capacity(env->src_rq) && - !is_big_task(p)) - return 0; - - if (env->flags & LBF_IGNORE_SMALL_TASKS && is_small_task(p)) - return 0; - - twf = task_will_fit(p, env->dst_cpu); - - /* - * Attempt to not pull tasks that don't fit. We may get lucky and find - * one that actually fits. - */ - if (env->flags & LBF_IGNORE_BIG_TASKS && !twf) - return 0; - - /* - * Group imbalance can sometimes cause work to be pulled across groups - * even though the group could have managed the imbalance on its own. - * Prevent inter-cluster migrations for big tasks when the number of - * tasks is lower than the capacity of the group. - */ - if (!twf && env->busiest_nr_running <= env->busiest_grp_capacity) - return 0; - if (!cpumask_test_cpu(env->dst_cpu, tsk_cpus_allowed(p))) { int cpu; schedstat_inc(p, se.statistics.nr_failed_migrations_affine); + env->flags |= LBF_SOME_PINNED; + /* * Remember if this task can be migrated to any other cpu in * our sched_group. We may want to revisit it if we couldn't @@ -6268,13 +6167,13 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) * Also avoid computing new_dst_cpu if we have already computed * one in current iteration. */ - if (!env->dst_grpmask || (env->flags & LBF_SOME_PINNED)) + if (!env->dst_grpmask || (env->flags & LBF_DST_PINNED)) return 0; /* Prevent to re-select dst_cpu via env's cpus */ for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) { if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) { - env->flags |= LBF_SOME_PINNED; + env->flags |= LBF_DST_PINNED; env->new_dst_cpu = cpu; break; } @@ -6293,19 +6192,20 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) /* * Aggressive migration if: - * 1) task is cache cold, or - * 2) too many balance attempts have failed. + * 1) destination numa is preferred + * 2) task is cache cold, or + * 3) too many balance attempts have failed. */ + tsk_cache_hot = task_hot(p, env); + if (!tsk_cache_hot) + tsk_cache_hot = migrate_degrades_locality(p, env); - tsk_cache_hot = task_hot(p, env->src_rq->clock_task, env->sd); - if (!tsk_cache_hot || - env->sd->nr_balance_failed > env->sd->cache_nice_tries) { - + if (migrate_improves_locality(p, env) || !tsk_cache_hot || + env->sd->nr_balance_failed > env->sd->cache_nice_tries) { if (tsk_cache_hot) { schedstat_inc(env->sd, lb_hot_gained[env->idle]); schedstat_inc(p, se.statistics.nr_forced_migrations); } - return 1; } @@ -6314,63 +6214,67 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) } /* - * move_one_task tries to move exactly one task from busiest to this_rq, as + * detach_task() -- detach the task for the migration specified in env + */ +static void detach_task(struct task_struct *p, struct lb_env *env) +{ + lockdep_assert_held(&env->src_rq->lock); + + deactivate_task(env->src_rq, p, 0); + p->on_rq = TASK_ON_RQ_MIGRATING; + set_task_cpu(p, env->dst_cpu); +} + +/* + * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as * part of active balancing operations within "domain". - * Returns 1 if successful and 0 otherwise. * - * Called with both runqueues locked. + * Returns a task if successful and NULL otherwise. */ -static int move_one_task(struct lb_env *env) +static struct task_struct *detach_one_task(struct lb_env *env) { struct task_struct *p, *n; + lockdep_assert_held(&env->src_rq->lock); + list_for_each_entry_safe(p, n, &env->src_rq->cfs_tasks, se.group_node) { if (!can_migrate_task(p, env)) continue; - move_task(p, env); + detach_task(p, env); + /* - * Right now, this is only the second place move_task() - * is called, so we can safely collect move_task() - * stats here rather than inside move_task(). + * Right now, this is only the second place where + * lb_gained[env->idle] is updated (other is detach_tasks) + * so we can safely collect stats here rather than + * inside detach_tasks(). */ schedstat_inc(env->sd, lb_gained[env->idle]); - per_cpu(dbs_boost_load_moved, env->dst_cpu) += pct_task_load(p); - - return 1; + return p; } - return 0; + return NULL; } -static unsigned long task_h_load(struct task_struct *p); - static const unsigned int sched_nr_migrate_break = 32; /* - * move_tasks tries to move up to imbalance weighted load from busiest to - * this_rq, as part of a balancing operation within domain "sd". - * Returns the number of pulled tasks if successful and 0 otherwise. + * detach_tasks() -- tries to detach up to imbalance weighted load from + * busiest_rq, as part of a balancing operation within domain "sd". * - * Called with both runqueues locked. + * Returns number of detached tasks if successful and 0 otherwise. */ -static int move_tasks(struct lb_env *env) +static int detach_tasks(struct lb_env *env) { struct list_head *tasks = &env->src_rq->cfs_tasks; struct task_struct *p; unsigned long load; - int pulled = 0; - int orig_loop = env->loop; + int detached = 0; + + lockdep_assert_held(&env->src_rq->lock); if (env->imbalance <= 0) return 0; - if (capacity(env->dst_rq) > capacity(env->src_rq)) - env->flags |= LBF_IGNORE_SMALL_TASKS; - else if (capacity(env->dst_rq) < capacity(env->src_rq) && - !sched_boost()) - env->flags |= LBF_IGNORE_BIG_TASKS; - -redo: while (!list_empty(tasks)) { p = list_first_entry(tasks, struct task_struct, se.group_node); @@ -6397,15 +6301,16 @@ static int move_tasks(struct lb_env *env) if ((load / 2) > env->imbalance) goto next; - move_task(p, env); - pulled++; + detach_task(p, env); + list_add(&p->se.group_node, &env->tasks); + + detached++; env->imbalance -= load; - per_cpu(dbs_boost_load_moved, env->dst_cpu) += pct_task_load(p); #ifdef CONFIG_PREEMPT /* * NEWIDLE balancing is a source of latency, so preemptible - * kernels will stop after the first task is pulled to minimize + * kernels will stop after the first task is detached to minimize * the critical section. */ if (env->idle == CPU_NEWLY_IDLE) @@ -6424,22 +6329,59 @@ static int move_tasks(struct lb_env *env) list_move_tail(&p->se.group_node, tasks); } - if (env->flags & (LBF_IGNORE_SMALL_TASKS | LBF_IGNORE_BIG_TASKS) - && !pulled) { - tasks = &env->src_rq->cfs_tasks; - env->flags &= ~(LBF_IGNORE_SMALL_TASKS | LBF_IGNORE_BIG_TASKS); - env->loop = orig_loop; - goto redo; - } - /* - * Right now, this is one of only two places move_task() is called, - * so we can safely collect move_task() stats here rather than - * inside move_task(). + * Right now, this is one of only two places we collect this stat + * so we can safely collect detach_one_task() stats here rather + * than inside detach_one_task(). */ - schedstat_add(env->sd, lb_gained[env->idle], pulled); + schedstat_add(env->sd, lb_gained[env->idle], detached); + + return detached; +} + +/* + * attach_task() -- attach the task detached by detach_task() to its new rq. + */ +static void attach_task(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_held(&rq->lock); + + BUG_ON(task_rq(p) != rq); + p->on_rq = TASK_ON_RQ_QUEUED; + activate_task(rq, p, 0); + check_preempt_curr(rq, p, 0); +} + +/* + * attach_one_task() -- attaches the task returned from detach_one_task() to + * its new rq. + */ +static void attach_one_task(struct rq *rq, struct task_struct *p) +{ + raw_spin_lock(&rq->lock); + attach_task(rq, p); + raw_spin_unlock(&rq->lock); +} - return pulled; +/* + * attach_tasks() -- attaches all tasks detached by detach_tasks() to their + * new rq. + */ +static void attach_tasks(struct lb_env *env) +{ + struct list_head *tasks = &env->tasks; + struct task_struct *p; + + raw_spin_lock(&env->dst_rq->lock); + + while (!list_empty(tasks)) { + p = list_first_entry(tasks, struct task_struct, se.group_node); + list_del_init(&p->se.group_node); + + attach_task(env->dst_rq, p); + } + + raw_spin_unlock(&env->dst_rq->lock); } #ifdef CONFIG_FAIR_GROUP_SCHED @@ -6501,104 +6443,63 @@ static void update_blocked_averages(int cpu) } /* - * Compute the cpu's hierarchical load factor for each task group. + * Compute the hierarchical load factor for cfs_rq and all its ascendants. * This needs to be done in a top-down fashion because the load of a child * group is a fraction of its parents load. */ -static int tg_load_down(struct task_group *tg, void *data) -{ - unsigned long load; - long cpu = (long)data; - - if (!tg->parent) { - load = cpu_rq(cpu)->load.weight; - } else { - load = tg->parent->cfs_rq[cpu]->h_load; - load *= tg->se[cpu]->load.weight; - load /= tg->parent->cfs_rq[cpu]->load.weight + 1; - } - - tg->cfs_rq[cpu]->h_load = load; - - return 0; -} - -static void update_h_load(long cpu) +static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) { - struct rq *rq = cpu_rq(cpu); + struct rq *rq = rq_of(cfs_rq); + struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; unsigned long now = jiffies; + unsigned long load; - if (rq->h_load_throttle == now) + if (cfs_rq->last_h_load_update == now) return; - rq->h_load_throttle = now; + cfs_rq->h_load_next = NULL; + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + cfs_rq->h_load_next = se; + if (cfs_rq->last_h_load_update == now) + break; + } - rcu_read_lock(); - walk_tg_tree(tg_load_down, tg_nop, (void *)cpu); - rcu_read_unlock(); + if (!se) { + cfs_rq->h_load = cfs_rq->runnable_load_avg; + cfs_rq->last_h_load_update = now; + } + + while ((se = cfs_rq->h_load_next) != NULL) { + load = cfs_rq->h_load; + load = div64_ul(load * se->avg.load_avg_contrib, + cfs_rq->runnable_load_avg + 1); + cfs_rq = group_cfs_rq(se); + cfs_rq->h_load = load; + cfs_rq->last_h_load_update = now; + } } static unsigned long task_h_load(struct task_struct *p) { struct cfs_rq *cfs_rq = task_cfs_rq(p); - unsigned long load; - load = p->se.load.weight; - load = div_u64(load * cfs_rq->h_load, cfs_rq->load.weight + 1); - - return load; + update_cfs_rq_h_load(cfs_rq); + return div64_ul(p->se.avg.load_avg_contrib * cfs_rq->h_load, + cfs_rq->runnable_load_avg + 1); } #else static inline void update_blocked_averages(int cpu) { } -static inline void update_h_load(long cpu) -{ -} - static unsigned long task_h_load(struct task_struct *p) { - return p->se.load.weight; + return p->se.avg.load_avg_contrib; } #endif /********** Helpers for find_busiest_group ************************/ -/* - * sd_lb_stats - Structure to store the statistics of a sched_domain - * during load balancing. - */ -struct sd_lb_stats { - struct sched_group *busiest; /* Busiest group in this sd */ - struct sched_group *this; /* Local group in this sd */ - unsigned long total_load; /* Total load of all groups in sd */ - unsigned long total_pwr; /* Total power of all groups in sd */ - unsigned long avg_load; /* Average load across all groups in sd */ - - /** Statistics of this group */ - unsigned long this_load; - unsigned long this_load_per_task; - unsigned long this_nr_running; - unsigned long this_has_capacity; - unsigned long this_group_capacity; - unsigned int this_idle_cpus; - - /* Statistics of the busiest group */ - unsigned int busiest_idle_cpus; - unsigned long max_load; - unsigned long busiest_load_per_task; - unsigned long busiest_nr_running; -#ifdef CONFIG_SCHED_HMP - unsigned long busiest_nr_small_tasks; - unsigned long busiest_nr_big_tasks; - u64 busiest_scaled_load; -#endif - unsigned long busiest_group_capacity; - unsigned long busiest_has_capacity; - unsigned int busiest_group_weight; - - int group_imb; /* Is there imbalance in this sd */ -}; /* * sg_lb_stats - stats of a sched_group required for load_balancing @@ -6606,57 +6507,64 @@ struct sd_lb_stats { struct sg_lb_stats { unsigned long avg_load; /*Avg load across the CPUs of the group */ unsigned long group_load; /* Total load over the CPUs of the group */ - unsigned long sum_nr_running; /* Nr tasks running in the group */ -#ifdef CONFIG_SCHED_HMP - unsigned long sum_nr_big_tasks, sum_nr_small_tasks; - u64 group_cpu_load; /* Scaled load of all CPUs of the group */ -#endif unsigned long sum_weighted_load; /* Weighted load of group's tasks */ + unsigned long load_per_task; unsigned long group_capacity; - unsigned long idle_cpus; - unsigned long group_weight; - int group_imb; /* Is there an imbalance in the group ? */ - int group_has_capacity; /* Is there extra capacity in the group? */ + unsigned long group_usage; /* Total usage of the group */ + unsigned int sum_nr_running; /* Nr tasks running in the group */ + unsigned int idle_cpus; + unsigned int group_weight; + enum group_type group_type; + int group_no_capacity; + int group_misfit_task; /* A cpu has a task too big for its capacity */ +#ifdef CONFIG_NUMA_BALANCING + unsigned int nr_numa_running; + unsigned int nr_preferred_running; +#endif }; -#ifdef CONFIG_SCHED_HMP - -static int -bail_inter_cluster_balance(struct lb_env *env, struct sd_lb_stats *sds) -{ - int nr_cpus; - - if (group_rq_capacity(sds->this) <= group_rq_capacity(sds->busiest)) - return 0; - - if (sds->busiest_nr_big_tasks) - return 0; - - nr_cpus = cpumask_weight(sched_group_cpus(sds->busiest)); - - if ((sds->busiest_scaled_load < nr_cpus * sched_spill_load) && - (sds->busiest_nr_running < - nr_cpus * sysctl_sched_spill_nr_run)) { - return 1; - } - - return 0; -} - -#else /* CONFIG_SCHED_HMP */ +/* + * sd_lb_stats - Structure to store the statistics of a sched_domain + * during load balancing. + */ +struct sd_lb_stats { + struct sched_group *busiest; /* Busiest group in this sd */ + struct sched_group *local; /* Local group in this sd */ + unsigned long total_load; /* Total load of all groups in sd */ + unsigned long total_capacity; /* Total capacity of all groups in sd */ + unsigned long avg_load; /* Average load across all groups in sd */ + + struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */ + struct sg_lb_stats local_stat; /* Statistics of the local group */ +}; -static inline int -bail_inter_cluster_balance(struct lb_env *env, struct sd_lb_stats *sds) +static inline void init_sd_lb_stats(struct sd_lb_stats *sds) { - return 0; + /* + * Skimp on the clearing to avoid duplicate work. We can avoid clearing + * local_stat because update_sg_lb_stats() does a full clear/assignment. + * We must however clear busiest_stat::avg_load because + * update_sd_pick_busiest() reads this before assignment. + */ + *sds = (struct sd_lb_stats){ + .busiest = NULL, + .local = NULL, + .total_load = 0UL, + .total_capacity = 0UL, + .busiest_stat = { + .avg_load = 0UL, + .sum_nr_running = 0, + .group_type = group_other, + }, + }; } -#endif /* CONFIG_SCHED_HMP */ - /** * get_sd_load_idx - Obtain the load index for a given sched domain. * @sd: The sched_domain whose load_idx is to be obtained. - * @idle: The Idle status of the CPU for whose sd load_icx is obtained. + * @idle: The idle status of the CPU for whose sd load_idx is obtained. + * + * Return: The load index. */ static inline int get_sd_load_idx(struct sched_domain *sd, enum cpu_idle_type idle) @@ -6679,35 +6587,21 @@ static inline int get_sd_load_idx(struct sched_domain *sd, return load_idx; } -static unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu) +static unsigned long default_scale_capacity(struct sched_domain *sd, int cpu) { return SCHED_CAPACITY_SCALE; } -unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu) -{ - return default_scale_freq_power(sd, cpu); -} - -static unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu) +unsigned long __weak arch_scale_freq_capacity(struct sched_domain *sd, int cpu) { - unsigned long weight = sd->span_weight; - unsigned long smt_gain = sd->smt_gain; - - smt_gain /= weight; - - return smt_gain; -} - -unsigned long __weak arch_scale_smt_power(struct sched_domain *sd, int cpu) -{ - return default_scale_smt_power(sd, cpu); + return default_scale_capacity(sd, cpu); } -static unsigned long scale_rt_power(int cpu) +static unsigned long scale_rt_capacity(int cpu) { struct rq *rq = cpu_rq(cpu); - u64 total, available, age_stamp, avg; + u64 total, used, age_stamp, avg; + s64 delta; /* * Since we're reading these variables without serialization make sure @@ -6716,122 +6610,222 @@ static unsigned long scale_rt_power(int cpu) age_stamp = ACCESS_ONCE(rq->age_stamp); avg = ACCESS_ONCE(rq->rt_avg); - total = sched_avg_period() + (rq->clock - age_stamp); + delta = rq_clock(rq) - age_stamp; + if (unlikely(delta < 0)) + delta = 0; - if (unlikely(total < avg)) { - /* Ensures that power won't end up being negative */ - available = 0; - } else { - available = total - avg; - } + total = sched_avg_period() + delta; - if (unlikely((s64)total < SCHED_CAPACITY_SCALE)) - total = SCHED_CAPACITY_SCALE; + used = div_u64(avg, total); - total >>= SCHED_CAPACITY_SHIFT; + if (likely(used < SCHED_CAPACITY_SCALE)) + return SCHED_CAPACITY_SCALE - used; - return div_u64(available, total); + return 1; } -static void update_cpu_power(struct sched_domain *sd, int cpu) +static void update_cpu_capacity(struct sched_domain *sd, int cpu) { - unsigned long weight = sd->span_weight; - unsigned long power = SCHED_CAPACITY_SCALE; + unsigned long capacity = arch_scale_cpu_capacity(sd, cpu); struct sched_group *sdg = sd->groups; + cpu_rq(cpu)->cpu_capacity_orig = capacity; - if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) { - if (sched_feat(ARCH_POWER)) - power *= arch_scale_smt_power(sd, cpu); - else - power *= default_scale_smt_power(sd, cpu); - - power >>= SCHED_CAPACITY_SHIFT; - } - - sdg->sgp->power_orig = power; - - if (sched_feat(ARCH_POWER)) - power *= arch_scale_freq_power(sd, cpu); - else - power *= default_scale_freq_power(sd, cpu); - - power >>= SCHED_CAPACITY_SHIFT; + capacity *= scale_rt_capacity(cpu); + capacity >>= SCHED_CAPACITY_SHIFT; - power *= scale_rt_power(cpu); - power >>= SCHED_CAPACITY_SHIFT; + if (!capacity) + capacity = 1; - if (!power) - power = 1; - - cpu_rq(cpu)->cpu_power = power; - sdg->sgp->power = power; + cpu_rq(cpu)->cpu_capacity = capacity; + sdg->sgc->capacity = capacity; + sdg->sgc->max_capacity = capacity; } -void update_group_power(struct sched_domain *sd, int cpu) +void update_group_capacity(struct sched_domain *sd, int cpu) { struct sched_domain *child = sd->child; struct sched_group *group, *sdg = sd->groups; - unsigned long power; + unsigned long capacity, max_capacity; unsigned long interval; interval = msecs_to_jiffies(sd->balance_interval); interval = clamp(interval, 1UL, max_load_balance_interval); - sdg->sgp->next_update = jiffies + interval; + sdg->sgc->next_update = jiffies + interval; if (!child) { - update_cpu_power(sd, cpu); + update_cpu_capacity(sd, cpu); return; } - power = 0; + capacity = 0; + max_capacity = 0; + + if (child->flags & SD_OVERLAP) { + /* + * SD_OVERLAP domains cannot assume that child groups + * span the current group. + */ + + for_each_cpu(cpu, sched_group_cpus(sdg)) { + struct sched_group_capacity *sgc; + struct rq *rq = cpu_rq(cpu); + + /* + * build_sched_domains() -> init_sched_groups_capacity() + * gets here before we've attached the domains to the + * runqueues. + * + * Use capacity_of(), which is set irrespective of domains + * in update_cpu_capacity(). + * + * This avoids capacity from being 0 and + * causing divide-by-zero issues on boot. + */ + if (unlikely(!rq->sd)) { + capacity += capacity_of(cpu); + } else { + sgc = rq->sd->groups->sgc; + capacity += sgc->capacity; + } + + max_capacity = max(capacity, max_capacity); + } + } else { + /* + * !SD_OVERLAP domains can assume that child groups + * span the current group. + */ + + group = child->groups; + do { + struct sched_group_capacity *sgc = group->sgc; + + capacity += sgc->capacity; + max_capacity = max(sgc->max_capacity, max_capacity); + group = group->next; + } while (group != child->groups); + } + + sdg->sgc->capacity = capacity; + sdg->sgc->max_capacity = max_capacity; +} + +/* + * Check whether the capacity of the rq has been noticeably reduced by side + * activity. The imbalance_pct is used for the threshold. + * Return true is the capacity is reduced + */ +static inline int +check_cpu_capacity(struct rq *rq, struct sched_domain *sd) +{ + return ((rq->cpu_capacity * sd->imbalance_pct) < + (rq->cpu_capacity_orig * 100)); +} + +/* + * Group imbalance indicates (and tries to solve) the problem where balancing + * groups is inadequate due to tsk_cpus_allowed() constraints. + * + * Imagine a situation of two groups of 4 cpus each and 4 tasks each with a + * cpumask covering 1 cpu of the first group and 3 cpus of the second group. + * Something like: + * + * { 0 1 2 3 } { 4 5 6 7 } + * * * * * + * + * If we were to balance group-wise we'd place two tasks in the first group and + * two tasks in the second group. Clearly this is undesired as it will overload + * cpu 3 and leave one of the cpus in the second group unused. + * + * The current solution to this issue is detecting the skew in the first group + * by noticing the lower domain failed to reach balance and had difficulty + * moving tasks due to affinity constraints. + * + * When this is so detected; this group becomes a candidate for busiest; see + * update_sd_pick_busiest(). And calculate_imbalance() and + * find_busiest_group() avoid some of the usual balance conditions to allow it + * to create an effective group imbalance. + * + * This is a somewhat tricky proposition since the next run might not find the + * group imbalance and decide the groups need to be balanced again. A most + * subtle and fragile situation. + */ + +static inline int sg_imbalanced(struct sched_group *group) +{ + return group->sgc->imbalance; +} + +/* + * group_has_capacity returns true if the group has spare capacity that could + * be used by some tasks. + * We consider that a group has spare capacity if the * number of task is + * smaller than the number of CPUs or if the usage is lower than the available + * capacity for CFS tasks. + * For the latter, we use a threshold to stabilize the state, to take into + * account the variance of the tasks' load and to return true if the available + * capacity in meaningful for the load balancer. + * As an example, an available capacity of 1% can appear but it doesn't make + * any benefit for the load balance. + */ +static inline bool +group_has_capacity(struct lb_env *env, struct sg_lb_stats *sgs) +{ + if (sgs->sum_nr_running < sgs->group_weight) + return true; + + if ((sgs->group_capacity * 100) > + (sgs->group_usage * env->sd->imbalance_pct)) + return true; - if (child->flags & SD_OVERLAP) { - /* - * SD_OVERLAP domains cannot assume that child groups - * span the current group. - */ + return false; +} - for_each_cpu(cpu, sched_group_cpus(sdg)) - power += power_of(cpu); - } else { - /* - * !SD_OVERLAP domains can assume that child groups - * span the current group. - */ +/* + * group_is_overloaded returns true if the group has more tasks than it can + * handle. + * group_is_overloaded is not equals to !group_has_capacity because a group + * with the exact right number of tasks, has no more spare capacity but is not + * overloaded so both group_has_capacity and group_is_overloaded return + * false. + */ +static inline bool +group_is_overloaded(struct lb_env *env, struct sg_lb_stats *sgs) +{ + if (sgs->sum_nr_running <= sgs->group_weight) + return false; - group = child->groups; - do { - power += group->sgp->power; - group = group->next; - } while (group != child->groups); - } + if ((sgs->group_capacity * 100) < + (sgs->group_usage * env->sd->imbalance_pct)) + return true; - sdg->sgp->power_orig = sdg->sgp->power = power; + return false; } /* - * Try and fix up capacity for tiny siblings, this is needed when - * things like SD_ASYM_PACKING need f_b_g to select another sibling - * which on its own isn't powerful enough. - * - * See update_sd_pick_busiest() and check_asym_packing(). + * group_smaller_cpu_capacity: Returns true if sched_group sg has smaller + * per-cpu capacity than sched_group ref. */ -static inline int -fix_small_capacity(struct sched_domain *sd, struct sched_group *group) +static inline bool +group_smaller_cpu_capacity(struct sched_group *sg, struct sched_group *ref) { - /* - * Only siblings can have significantly less than SCHED_CAPACITY_SCALE - */ - if (!(sd->flags & SD_SHARE_CPUPOWER)) - return 0; + return sg->sgc->max_capacity + capacity_margin - SCHED_LOAD_SCALE < + ref->sgc->max_capacity; +} - /* - * If ~90% of the cpu_power is still there, we're good. - */ - if (group->sgp->power * 32 > group->sgp->power_orig * 29) - return 1; +static enum group_type group_classify(struct lb_env *env, + struct sched_group *group, + struct sg_lb_stats *sgs) +{ + if (sgs->group_no_capacity) + return group_overloaded; - return 0; + if (sg_imbalanced(group)) + return group_imbalanced; + + if (sgs->group_misfit_task) + return group_misfit_task; + return group_other; } /** @@ -6840,148 +6834,63 @@ fix_small_capacity(struct sched_domain *sd, struct sched_group *group) * @group: sched_group whose statistics are to be updated. * @load_idx: Load index of sched_domain of this_cpu for load calc. * @local_group: Does group contain this_cpu. - * @balance: Should we balance. * @sgs: variable to hold the statistics for this group. + * @overload: Indicate more than one runnable task for any CPU. + * @overutilized: Indicate overutilization for any CPU. */ static inline void update_sg_lb_stats(struct lb_env *env, struct sched_group *group, int load_idx, - int local_group, int *balance, struct sg_lb_stats *sgs) + int local_group, struct sg_lb_stats *sgs, + bool *overload, bool *overutilized) { - unsigned long nr_running, max_nr_running, min_nr_running; - unsigned long load, max_cpu_load, min_cpu_load; - unsigned int balance_cpu = -1, first_idle_cpu = 0; - unsigned long avg_load_per_task = 0; + unsigned long load; int i; - if (local_group) - balance_cpu = group_balance_cpu(group); - - /* Tally up the load of all CPUs in the group */ - max_cpu_load = 0; - min_cpu_load = ~0UL; - max_nr_running = 0; - min_nr_running = ~0UL; + memset(sgs, 0, sizeof(*sgs)); for_each_cpu_and(i, sched_group_cpus(group), env->cpus) { struct rq *rq = cpu_rq(i); - trace_sched_cpu_load(cpu_rq(i), idle_cpu(i), - mostly_idle_cpu(i), - sched_irqload(i), - power_cost_at_freq(i, 0), - cpu_temp(i)); - nr_running = rq->nr_running; - /* Bias balancing toward cpus of our domain */ - if (local_group) { - if (idle_cpu(i) && !first_idle_cpu && - cpumask_test_cpu(i, sched_group_mask(group))) { - first_idle_cpu = 1; - balance_cpu = i; - } - + if (local_group) load = target_load(i, load_idx); - } else { + else load = source_load(i, load_idx); - if (load > max_cpu_load) - max_cpu_load = load; - if (min_cpu_load > load) - min_cpu_load = load; - - if (nr_running > max_nr_running) - max_nr_running = nr_running; - if (min_nr_running > nr_running) - min_nr_running = nr_running; - } sgs->group_load += load; - sgs->sum_nr_running += nr_running; -#ifdef CONFIG_SCHED_HMP - sgs->sum_nr_big_tasks += rq->hmp_stats.nr_big_tasks; - sgs->sum_nr_small_tasks += rq->hmp_stats.nr_small_tasks; - sgs->group_cpu_load += cpu_load(i); + sgs->group_usage += get_cpu_usage(i); + sgs->sum_nr_running += rq->cfs.h_nr_running; + + if (rq->nr_running > 1) + *overload = true; + +#ifdef CONFIG_NUMA_BALANCING + sgs->nr_numa_running += rq->nr_numa_running; + sgs->nr_preferred_running += rq->nr_preferred_running; #endif sgs->sum_weighted_load += weighted_cpuload(i); if (idle_cpu(i)) sgs->idle_cpus++; - } - /* - * First idle cpu or the first cpu(busiest) in this sched group - * is eligible for doing load balancing at this and above - * domains. In the newly idle case, we will allow all the cpu's - * to do the newly idle load balance. - */ - if (local_group) { - if (env->idle != CPU_NEWLY_IDLE) { - if (balance_cpu != env->dst_cpu) { - *balance = 0; - return; - } - update_group_power(env->sd, env->dst_cpu); - } else if (time_after_eq(jiffies, group->sgp->next_update)) - update_group_power(env->sd, env->dst_cpu); + if (cpu_overutilized(i)) { + *overutilized = true; + if (!sgs->group_misfit_task && rq->misfit_task) + sgs->group_misfit_task = capacity_of(i); + } } - /* Adjust by relative CPU power of the group */ - sgs->avg_load = (sgs->group_load*SCHED_CAPACITY_SCALE) / group->sgp->power; + /* Adjust by relative CPU capacity of the group */ + sgs->group_capacity = group->sgc->capacity; + sgs->avg_load = (sgs->group_load*SCHED_CAPACITY_SCALE) / sgs->group_capacity; - /* - * Consider the group unbalanced when the imbalance is larger - * than the average weight of a task. - * - * APZ: with cgroup the avg task weight can vary wildly and - * might not be a suitable number - should we keep a - * normalized nr_running number somewhere that negates - * the hierarchy? - */ if (sgs->sum_nr_running) - avg_load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running; + sgs->load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running; - if ((max_cpu_load - min_cpu_load) >= avg_load_per_task && - (max_nr_running - min_nr_running) > 1) - sgs->group_imb = 1; - - sgs->group_capacity = DIV_ROUND_CLOSEST(group->sgp->power, - SCHED_CAPACITY_SCALE); - if (!sgs->group_capacity) - sgs->group_capacity = fix_small_capacity(env->sd, group); sgs->group_weight = group->group_weight; - if (sgs->group_capacity > sgs->sum_nr_running) - sgs->group_has_capacity = 1; -} - -#ifdef CONFIG_SCHED_HMP -static bool update_sd_pick_busiest_active_balance(struct lb_env *env, - struct sd_lb_stats *sds, - struct sched_group *sg, - struct sg_lb_stats *sgs) -{ - if (env->idle != CPU_NOT_IDLE && - capacity(env->dst_rq) > group_rq_capacity(sg)) { - if (sched_boost() && !sds->busiest && sgs->sum_nr_running) { - env->flags |= LBF_SCHED_BOOST_ACTIVE_BALANCE; - return true; - } - - if (sgs->sum_nr_big_tasks > sds->busiest_nr_big_tasks) { - env->flags |= LBF_BIG_TASK_ACTIVE_BALANCE; - return true; - } - } - - return false; + sgs->group_no_capacity = group_is_overloaded(env, sgs); + sgs->group_type = group_classify(env, group, sgs); } -#else -static bool update_sd_pick_busiest_active_balance(struct lb_env *env, - struct sd_lb_stats *sds, - struct sched_group *sg, - struct sg_lb_stats *sgs) -{ - return false; -} -#endif /** * update_sd_pick_busiest - return 1 on busiest group @@ -6992,67 +6901,52 @@ static bool update_sd_pick_busiest_active_balance(struct lb_env *env, * * Determine if @sg is a busier group than the previously selected * busiest group. + * + * Return: %true if @sg is a busier group than the previously selected + * busiest group. %false otherwise. */ static bool update_sd_pick_busiest(struct lb_env *env, struct sd_lb_stats *sds, struct sched_group *sg, struct sg_lb_stats *sgs) { - int cpu, cpu_busiest; - unsigned int pc, pc_busiest; + struct sg_lb_stats *busiest = &sds->busiest_stat; - if (update_sd_pick_busiest_active_balance(env, sds, sg, sgs)) + if (sgs->group_type > busiest->group_type) return true; - if (sgs->avg_load < sds->max_load) + if (sgs->group_type < busiest->group_type) return false; - if (sgs->sum_nr_running > sgs->group_capacity) { - env->flags &= ~LBF_EA_ACTIVE_BALANCE; - return true; - } + /* + * Candidate sg doesn't face any serious load-balance problems + * so don't pick it if the local sg is already filled up. + */ + if (sgs->group_type == group_other && + !group_has_capacity(env, &sds->local_stat)) + return false; - if (sgs->group_imb) { - env->flags &= ~LBF_EA_ACTIVE_BALANCE; - return true; - } + if (sgs->avg_load <= busiest->avg_load) + return false; /* - * Mark a less power-efficient CPU as busy only we are close to - * throttling. We want to prioritize spreading work over power - * optimization. - */ - cpu = group_first_cpu(sg); - if (sysctl_sched_enable_power_aware && - (!sds->busiest || (env->flags & LBF_EA_ACTIVE_BALANCE)) && - (capacity(env->dst_rq) == group_rq_capacity(sg)) && - sgs->sum_nr_running && (env->idle != CPU_NOT_IDLE) && - !is_task_migration_throttled(cpu_rq(cpu)->curr) && - is_cpu_throttling_imminent(cpu)) { - pc = power_cost_at_freq(cpu, 0); - if (sds->busiest) { - cpu_busiest = group_first_cpu(sds->busiest); - pc_busiest = power_cost_at_freq(cpu_busiest, 0); - if (pc_busiest < pc) - return true; - } else { - if (power_cost_at_freq(env->dst_cpu, 0) < pc) { - env->flags |= LBF_EA_ACTIVE_BALANCE; - return true; - } - } - } - - if (sgs->avg_load == sds->max_load) + * Candiate sg has no more than one task per cpu and has higher + * per-cpu capacity. No reason to pull tasks to less capable cpus. + */ + if (sgs->sum_nr_running <= sgs->group_weight && + group_smaller_cpu_capacity(sds->local, sg)) return false; + /* This is the busiest node in its class. */ + if (!(env->sd->flags & SD_ASYM_PACKING)) + return true; + /* * ASYM_PACKING needs to move all the work to the lowest * numbered CPUs in the group, therefore mark all groups * higher than ourself as busy. */ - if ((env->sd->flags & SD_ASYM_PACKING) && sgs->sum_nr_running && - env->dst_cpu < group_first_cpu(sg)) { + if (sgs->sum_nr_running && env->dst_cpu < group_first_cpu(sg)) { if (!sds->busiest) return true; @@ -7063,19 +6957,48 @@ static bool update_sd_pick_busiest(struct lb_env *env, return false; } +#ifdef CONFIG_NUMA_BALANCING +static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) +{ + if (sgs->sum_nr_running > sgs->nr_numa_running) + return regular; + if (sgs->sum_nr_running > sgs->nr_preferred_running) + return remote; + return all; +} + +static inline enum fbq_type fbq_classify_rq(struct rq *rq) +{ + if (rq->nr_running > rq->nr_numa_running) + return regular; + if (rq->nr_running > rq->nr_preferred_running) + return remote; + return all; +} +#else +static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) +{ + return all; +} + +static inline enum fbq_type fbq_classify_rq(struct rq *rq) +{ + return regular; +} +#endif /* CONFIG_NUMA_BALANCING */ + /** * update_sd_lb_stats - Update sched_domain's statistics for load balancing. * @env: The load balancing environment. - * @balance: Should we balance. * @sds: variable to hold the statistics for this sched_domain. */ -static inline void update_sd_lb_stats(struct lb_env *env, - int *balance, struct sd_lb_stats *sds) +static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds) { struct sched_domain *child = env->sd->child; struct sched_group *sg = env->sd->groups; - struct sg_lb_stats sgs; + struct sg_lb_stats tmp_sgs; int load_idx, prefer_sibling = 0; + bool overload = false, overutilized = false; if (child && child->flags & SD_PREFER_SIBLING) prefer_sibling = 1; @@ -7083,60 +7006,72 @@ static inline void update_sd_lb_stats(struct lb_env *env, load_idx = get_sd_load_idx(env->sd, env->idle); do { + struct sg_lb_stats *sgs = &tmp_sgs; int local_group; local_group = cpumask_test_cpu(env->dst_cpu, sched_group_cpus(sg)); - memset(&sgs, 0, sizeof(sgs)); - update_sg_lb_stats(env, sg, load_idx, local_group, balance, &sgs); + if (local_group) { + sds->local = sg; + sgs = &sds->local_stat; - if (local_group && !(*balance)) - return; + if (env->idle != CPU_NEWLY_IDLE || + time_after_eq(jiffies, sg->sgc->next_update)) + update_group_capacity(env->sd, env->dst_cpu); + } - sds->total_load += sgs.group_load; - sds->total_pwr += sg->sgp->power; + update_sg_lb_stats(env, sg, load_idx, local_group, sgs, + &overload, &overutilized); + + if (local_group) + goto next_group; /* * In case the child domain prefers tasks go to siblings - * first, lower the sg capacity to one so that we'll try + * first, lower the sg capacity so that we'll try * and move all the excess tasks away. We lower the capacity * of a group only if the local group has the capacity to fit - * these excess tasks, i.e. nr_running < group_capacity. The - * extra check prevents the case where you always pull from the - * heaviest group when it is already under-utilized (possible - * with a large weight task outweighs the tasks on the system). + * these excess tasks. The extra check prevents the case where + * you always pull from the heaviest group when it is already + * under-utilized (possible with a large weight task outweighs + * the tasks on the system). */ - if (prefer_sibling && !local_group && sds->this_has_capacity) - sgs.group_capacity = min(sgs.group_capacity, 1UL); + if (prefer_sibling && sds->local && + group_has_capacity(env, &sds->local_stat) && + (sgs->sum_nr_running > 1)) { + sgs->group_no_capacity = 1; + sgs->group_type = group_overloaded; + } - if (local_group) { - sds->this_load = sgs.avg_load; - sds->this = sg; - sds->this_nr_running = sgs.sum_nr_running; - sds->this_load_per_task = sgs.sum_weighted_load; - sds->this_has_capacity = sgs.group_has_capacity; - sds->this_idle_cpus = sgs.idle_cpus; - sds->this_group_capacity = sgs.group_capacity; - } else if (update_sd_pick_busiest(env, sds, sg, &sgs)) { - sds->max_load = sgs.avg_load; + if (update_sd_pick_busiest(env, sds, sg, sgs)) { sds->busiest = sg; - env->busiest_nr_running = sds->busiest_nr_running - = sgs.sum_nr_running; - sds->busiest_idle_cpus = sgs.idle_cpus; - env->busiest_grp_capacity = sds->busiest_group_capacity - = sgs.group_capacity; - sds->busiest_load_per_task = sgs.sum_weighted_load; - sds->busiest_has_capacity = sgs.group_has_capacity; - sds->busiest_group_weight = sgs.group_weight; - sds->group_imb = sgs.group_imb; -#ifdef CONFIG_SCHED_HMP - sds->busiest_nr_small_tasks = sgs.sum_nr_small_tasks; - sds->busiest_nr_big_tasks = sgs.sum_nr_big_tasks; - sds->busiest_scaled_load = sgs.group_cpu_load; -#endif + sds->busiest_stat = *sgs; } +next_group: + /* Now, start updating sd_lb_stats */ + sds->total_load += sgs->group_load; + sds->total_capacity += sgs->group_capacity; + sg = sg->next; } while (sg != env->sd->groups); + + if (env->sd->flags & SD_NUMA) + env->fbq_type = fbq_classify_group(&sds->busiest_stat); + + env->src_grp_nr_running = sds->busiest_stat.sum_nr_running; + + if (!env->sd->parent) { + /* update overload indicator if we are at root domain */ + if (env->dst_rq->rd->overload != overload) + env->dst_rq->rd->overload = overload; + + /* Update over-utilization (tipping point, U >= 0) indicator */ + if (env->dst_rq->rd->overutilized != overutilized) + env->dst_rq->rd->overutilized = overutilized; + } else { + if (!env->dst_rq->rd->overutilized && overutilized) + env->dst_rq->rd->overutilized = true; + } } /** @@ -7156,7 +7091,7 @@ static inline void update_sd_lb_stats(struct lb_env *env, * assuming lower CPU number will be equivalent to lower a SMT thread * number. * - * Returns 1 when packing is required and a task should be moved to + * Return: 1 when packing is required and a task should be moved to * this CPU. The amount of the imbalance is returned in *imbalance. * * @env: The load balancing environment. @@ -7177,7 +7112,8 @@ static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds) return 0; env->imbalance = DIV_ROUND_CLOSEST( - sds->max_load * sds->busiest->sgp->power, SCHED_CAPACITY_SCALE); + sds->busiest_stat.avg_load * sds->busiest_stat.group_capacity, + SCHED_CAPACITY_SCALE); return 1; } @@ -7192,64 +7128,64 @@ static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds) static inline void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds) { - unsigned long tmp, pwr_now = 0, pwr_move = 0; + unsigned long tmp, capa_now = 0, capa_move = 0; unsigned int imbn = 2; unsigned long scaled_busy_load_per_task; + struct sg_lb_stats *local, *busiest; - if (sds->this_nr_running) { - sds->this_load_per_task /= sds->this_nr_running; - if (sds->busiest_load_per_task > - sds->this_load_per_task) - imbn = 1; - } else { - sds->this_load_per_task = - cpu_avg_load_per_task(env->dst_cpu); - } + local = &sds->local_stat; + busiest = &sds->busiest_stat; + + if (!local->sum_nr_running) + local->load_per_task = cpu_avg_load_per_task(env->dst_cpu); + else if (busiest->load_per_task > local->load_per_task) + imbn = 1; - scaled_busy_load_per_task = sds->busiest_load_per_task - * SCHED_CAPACITY_SCALE; - scaled_busy_load_per_task /= sds->busiest->sgp->power; + scaled_busy_load_per_task = + (busiest->load_per_task * SCHED_CAPACITY_SCALE) / + busiest->group_capacity; - if (sds->max_load - sds->this_load + scaled_busy_load_per_task >= - (scaled_busy_load_per_task * imbn)) { - env->imbalance = sds->busiest_load_per_task; + if (busiest->avg_load + scaled_busy_load_per_task >= + local->avg_load + (scaled_busy_load_per_task * imbn)) { + env->imbalance = busiest->load_per_task; return; } /* * OK, we don't have enough imbalance to justify moving tasks, - * however we may be able to increase total CPU power used by + * however we may be able to increase total CPU capacity used by * moving them. */ - pwr_now += sds->busiest->sgp->power * - min(sds->busiest_load_per_task, sds->max_load); - pwr_now += sds->this->sgp->power * - min(sds->this_load_per_task, sds->this_load); - pwr_now /= SCHED_CAPACITY_SCALE; + capa_now += busiest->group_capacity * + min(busiest->load_per_task, busiest->avg_load); + capa_now += local->group_capacity * + min(local->load_per_task, local->avg_load); + capa_now /= SCHED_CAPACITY_SCALE; /* Amount of load we'd subtract */ - tmp = (sds->busiest_load_per_task * SCHED_CAPACITY_SCALE) / - sds->busiest->sgp->power; - if (sds->max_load > tmp) - pwr_move += sds->busiest->sgp->power * - min(sds->busiest_load_per_task, sds->max_load - tmp); + if (busiest->avg_load > scaled_busy_load_per_task) { + capa_move += busiest->group_capacity * + min(busiest->load_per_task, + busiest->avg_load - scaled_busy_load_per_task); + } /* Amount of load we'd add */ - if (sds->max_load * sds->busiest->sgp->power < - sds->busiest_load_per_task * SCHED_CAPACITY_SCALE) - tmp = (sds->max_load * sds->busiest->sgp->power) / - sds->this->sgp->power; - else - tmp = (sds->busiest_load_per_task * SCHED_CAPACITY_SCALE) / - sds->this->sgp->power; - pwr_move += sds->this->sgp->power * - min(sds->this_load_per_task, sds->this_load + tmp); - pwr_move /= SCHED_CAPACITY_SCALE; + if (busiest->avg_load * busiest->group_capacity < + busiest->load_per_task * SCHED_CAPACITY_SCALE) { + tmp = (busiest->avg_load * busiest->group_capacity) / + local->group_capacity; + } else { + tmp = (busiest->load_per_task * SCHED_CAPACITY_SCALE) / + local->group_capacity; + } + capa_move += local->group_capacity * + min(local->load_per_task, local->avg_load + tmp); + capa_move /= SCHED_CAPACITY_SCALE; /* Move if we gain throughput */ - if (pwr_move > pwr_now) - env->imbalance = sds->busiest_load_per_task; + if (capa_move > capa_now) + env->imbalance = busiest->load_per_task; } /** @@ -7261,33 +7197,58 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds) static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) { unsigned long max_pull, load_above_capacity = ~0UL; + struct sg_lb_stats *local, *busiest; - sds->busiest_load_per_task /= sds->busiest_nr_running; - if (sds->group_imb) { - sds->busiest_load_per_task = - min(sds->busiest_load_per_task, sds->avg_load); + local = &sds->local_stat; + busiest = &sds->busiest_stat; + + if (busiest->group_type == group_imbalanced) { + /* + * In the group_imb case we cannot rely on group-wide averages + * to ensure cpu-load equilibrium, look at wider averages. XXX + */ + busiest->load_per_task = + min(busiest->load_per_task, sds->avg_load); } /* * In the presence of smp nice balancing, certain scenarios can have * max load less than avg load(as we skip the groups at or below - * its cpu_power, while calculating max_load..) + * its cpu_capacity, while calculating max_load..) */ - if (sds->max_load < sds->avg_load) { - env->imbalance = 0; - return fix_small_imbalance(env, sds); - } + if (busiest->avg_load <= sds->avg_load || + local->avg_load >= sds->avg_load) { + /* Misfitting tasks should be migrated in any case */ + if (busiest->group_type == group_misfit_task) { + env->imbalance = busiest->group_misfit_task; + return; + } - if (!sds->group_imb) { /* - * Don't want to pull so many tasks that a group would go idle. + * Busiest group is overloaded, local is not, use the spare + * cycles to maximize throughput */ - load_above_capacity = (sds->busiest_nr_running - - sds->busiest_group_capacity); + if (busiest->group_type == group_overloaded && + local->group_type <= group_misfit_task) { + env->imbalance = busiest->load_per_task; + return; + } - load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_CAPACITY_SCALE); + env->imbalance = 0; + return fix_small_imbalance(env, sds); + } - load_above_capacity /= sds->busiest->sgp->power; + /* + * If there aren't any idle cpus, avoid creating some. + */ + if (busiest->group_type == group_overloaded && + local->group_type == group_overloaded) { + load_above_capacity = busiest->sum_nr_running * + SCHED_LOAD_SCALE; + if (load_above_capacity > busiest->group_capacity) + load_above_capacity -= busiest->group_capacity; + else + load_above_capacity = ~0UL; } /* @@ -7297,15 +7258,19 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s * we also don't want to reduce the group load below the group capacity * (so that we can implement power-savings policies etc). Thus we look * for the minimum possible imbalance. - * Be careful of negative numbers as they'll appear as very large values - * with unsigned longs. */ - max_pull = min(sds->max_load - sds->avg_load, load_above_capacity); + max_pull = min(busiest->avg_load - sds->avg_load, load_above_capacity); /* How much load to actually move to equalise the imbalance */ - env->imbalance = min(max_pull * sds->busiest->sgp->power, - (sds->avg_load - sds->this_load) * sds->this->sgp->power) - / SCHED_CAPACITY_SCALE; + env->imbalance = min( + max_pull * busiest->group_capacity, + (sds->avg_load - local->avg_load) * local->group_capacity + ) / SCHED_CAPACITY_SCALE; + + /* Boost imbalance to allow misfit task to be balanced. */ + if (busiest->group_type == group_misfit_task) + env->imbalance = max_t(long, env->imbalance, + busiest->group_misfit_task); /* * if *imbalance is less than the average load per runnable task @@ -7313,9 +7278,8 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s * a think about bumping its value to force at least one task to be * moved */ - if (env->imbalance < sds->busiest_load_per_task) + if (env->imbalance < busiest->load_per_task) return fix_small_imbalance(env, sds); - } /******* find_busiest_group() helpers end here *********************/ @@ -7331,158 +7295,108 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s * to restore balance. * * @env: The load balancing environment. - * @balance: Pointer to a variable indicating if this_cpu - * is the appropriate cpu to perform load balancing at this_level. * - * Returns: - the busiest group if imbalance exists. + * Return: - The busiest group if imbalance exists. * - If no imbalance and user has opted for power-savings balance, * return the least loaded group whose CPUs can be * put to idle by rebalancing its tasks onto our group. */ -static struct sched_group * -find_busiest_group(struct lb_env *env, int *balance) +static struct sched_group *find_busiest_group(struct lb_env *env) { + struct sg_lb_stats *local, *busiest; struct sd_lb_stats sds; - memset(&sds, 0, sizeof(sds)); + init_sd_lb_stats(&sds); /* * Compute the various statistics relavent for load balancing at * this level. */ - update_sd_lb_stats(env, balance, &sds); + update_sd_lb_stats(env, &sds); - /* - * this_cpu is not the appropriate cpu to perform load balancing at - * this level. - */ - if (!(*balance)) - goto ret; + if (energy_aware() && !env->dst_rq->rd->overutilized) + goto out_balanced; + + local = &sds.local_stat; + busiest = &sds.busiest_stat; + /* ASYM feature bypasses nice load balance check */ if ((env->idle == CPU_IDLE || env->idle == CPU_NEWLY_IDLE) && check_asym_packing(env, &sds)) return sds.busiest; /* There is no busy sibling group to pull tasks from */ - if (!sds.busiest || sds.busiest_nr_running == 0) - goto out_balanced; - - if (env->flags & LBF_HMP_ACTIVE_BALANCE) - goto force_balance; - - if (bail_inter_cluster_balance(env, &sds)) + if (!sds.busiest || busiest->sum_nr_running == 0) goto out_balanced; - sds.avg_load = (SCHED_CAPACITY_SCALE * sds.total_load) / sds.total_pwr; + sds.avg_load = (SCHED_CAPACITY_SCALE * sds.total_load) + / sds.total_capacity; /* * If the busiest group is imbalanced the below checks don't - * work because they assumes all things are equal, which typically + * work because they assume all things are equal, which typically * isn't true due to cpus_allowed constraints and the like. */ - if (sds.group_imb) + if (busiest->group_type == group_imbalanced) goto force_balance; /* SD_BALANCE_NEWIDLE trumps SMP nice when underutilized */ - if (env->idle == CPU_NEWLY_IDLE && sds.this_has_capacity && - !sds.busiest_has_capacity) + if (env->idle == CPU_NEWLY_IDLE && group_has_capacity(env, local) && + busiest->group_no_capacity) + goto force_balance; + + /* Misfitting tasks should be dealt with regardless of the avg load */ + if (busiest->group_type == group_misfit_task) { goto force_balance; + } /* - * If the local group is more busy than the selected busiest group + * If the local group is busier than the selected busiest group * don't try and pull any tasks. */ - if (sds.this_load >= sds.max_load) + if (local->avg_load >= busiest->avg_load) goto out_balanced; /* * Don't pull any tasks if this group is already above the domain * average load. */ - if (sds.this_load >= sds.avg_load) + if (local->avg_load >= sds.avg_load) goto out_balanced; if (env->idle == CPU_IDLE) { /* - * This cpu is idle. If the busiest group load doesn't - * have more tasks than the number of available cpu's and - * there is no imbalance between this and busiest group - * wrt to idle cpu's, it is balanced. + * This cpu is idle. If the busiest group is not overloaded + * and there is no imbalance between this and busiest group + * wrt idle cpus, it is balanced. The imbalance becomes + * significant if the diff is greater than 1 otherwise we + * might end up to just move the imbalance on another group */ - if ((sds.this_idle_cpus <= sds.busiest_idle_cpus + 1) && - sds.busiest_nr_running <= sds.busiest_group_weight) + if ((busiest->group_type != group_overloaded) && + (local->idle_cpus <= (busiest->idle_cpus + 1)) && + !group_smaller_cpu_capacity(sds.busiest, sds.local)) goto out_balanced; } else { /* * In the CPU_NEWLY_IDLE, CPU_NOT_IDLE cases, use * imbalance_pct to be conservative. */ - if (100 * sds.max_load <= env->sd->imbalance_pct * sds.this_load) + if (100 * busiest->avg_load <= + env->sd->imbalance_pct * local->avg_load) goto out_balanced; } force_balance: + env->busiest_group_type = busiest->group_type; /* Looks like there is an imbalance. Compute it */ calculate_imbalance(env, &sds); return sds.busiest; out_balanced: -ret: env->imbalance = 0; return NULL; } -#ifdef CONFIG_SCHED_HMP -static struct rq *find_busiest_queue_hmp(struct lb_env *env, - struct sched_group *group) -{ - struct rq *busiest = NULL, *busiest_big = NULL; - u64 max_runnable_avg = 0, max_runnable_avg_big = 0; - int max_nr_big = 0, nr_big; - bool find_big = !!(env->flags & LBF_BIG_TASK_ACTIVE_BALANCE); - int i; - - for_each_cpu(i, sched_group_cpus(group)) { - struct rq *rq = cpu_rq(i); - u64 cumulative_runnable_avg = - rq->hmp_stats.cumulative_runnable_avg; - - if (!cpumask_test_cpu(i, env->cpus)) - continue; - - - if (find_big) { - nr_big = nr_big_tasks(rq); - if (nr_big > max_nr_big || - (nr_big > 0 && nr_big == max_nr_big && - cumulative_runnable_avg > max_runnable_avg_big)) { - max_runnable_avg_big = cumulative_runnable_avg; - busiest_big = rq; - max_nr_big = nr_big; - continue; - } - } - - if (cumulative_runnable_avg > max_runnable_avg) { - max_runnable_avg = cumulative_runnable_avg; - busiest = rq; - } - } - - if (busiest_big) - return busiest_big; - - env->flags &= ~LBF_BIG_TASK_ACTIVE_BALANCE; - return busiest; -} -#else -static inline struct rq *find_busiest_queue_hmp(struct lb_env *env, - struct sched_group *group) -{ - return NULL; -} -#endif - /* * find_busiest_queue - find the busiest runqueue among the cpus in group. */ @@ -7490,44 +7404,65 @@ static struct rq *find_busiest_queue(struct lb_env *env, struct sched_group *group) { struct rq *busiest = NULL, *rq; - unsigned long max_load = 0; + unsigned long busiest_load = 0, busiest_capacity = 1; int i; - if (sched_enable_hmp) - return find_busiest_queue_hmp(env, group); - - for_each_cpu(i, sched_group_cpus(group)) { - unsigned long power = power_of(i); - unsigned long capacity = DIV_ROUND_CLOSEST(power, - SCHED_CAPACITY_SCALE); - unsigned long wl; + for_each_cpu_and(i, sched_group_cpus(group), env->cpus) { + unsigned long capacity, wl; + enum fbq_type rt; - if (!capacity) - capacity = fix_small_capacity(env->sd, group); + rq = cpu_rq(i); + rt = fbq_classify_rq(rq); - if (!cpumask_test_cpu(i, env->cpus)) + /* + * We classify groups/runqueues into three groups: + * - regular: there are !numa tasks + * - remote: there are numa tasks that run on the 'wrong' node + * - all: there is no distinction + * + * In order to avoid migrating ideally placed numa tasks, + * ignore those when there's better options. + * + * If we ignore the actual busiest queue to migrate another + * task, the next balance pass can still reduce the busiest + * queue by moving tasks around inside the node. + * + * If we cannot move enough load due to this classification + * the next pass will adjust the group classification and + * allow migration of more tasks. + * + * Both cases only affect the total convergence complexity. + */ + if (rt > env->fbq_type) continue; - rq = cpu_rq(i); + capacity = capacity_of(i); + wl = weighted_cpuload(i); /* * When comparing with imbalance, use weighted_cpuload() - * which is not scaled with the cpu power. + * which is not scaled with the cpu capacity. */ - if (capacity && rq->nr_running == 1 && wl > env->imbalance) + if (rq->nr_running == 1 && wl > env->imbalance && + !check_cpu_capacity(rq, env->sd) && + env->busiest_group_type != group_misfit_task) continue; /* * For the load comparisons with the other cpu's, consider - * the weighted_cpuload() scaled with the cpu power, so that - * the load can be moved away from the cpu that is potentially - * running at a lower capacity. + * the weighted_cpuload() scaled with the cpu capacity, so + * that the load can be moved away from the cpu that is + * potentially running at a lower capacity. + * + * Thus we're looking for max(wl_i / capacity_i), crosswise + * multiplication to rid ourselves of the division works out + * to: wl_i * capacity_j > wl_j * capacity_i; where j is + * our previous maximum. */ - wl = (wl * SCHED_CAPACITY_SCALE) / power; - - if (wl > max_load) { - max_load = wl; + if (wl * busiest_capacity > busiest_load * capacity) { + busiest_load = wl; + busiest_capacity = capacity; busiest = rq; } } @@ -7539,7 +7474,7 @@ static struct rq *find_busiest_queue(struct lb_env *env, * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but * so long as it is large enough. */ -#define MAX_PINNED_INTERVAL 16 +#define MAX_PINNED_INTERVAL 512 /* Working cpumask for load_balance and load_balance_newidle. */ DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); @@ -7548,9 +7483,6 @@ static int need_active_balance(struct lb_env *env) { struct sched_domain *sd = env->sd; - if (env->flags & LBF_HMP_ACTIVE_BALANCE) - return 1; - if (env->idle == CPU_NEWLY_IDLE) { /* @@ -7562,35 +7494,90 @@ static int need_active_balance(struct lb_env *env) return 1; } + /* + * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task. + * It's worth migrating the task if the src_cpu's capacity is reduced + * because of other sched_class or IRQs if more capacity stays + * available on dst_cpu. + */ + if ((env->idle != CPU_NOT_IDLE) && + (env->src_rq->cfs.h_nr_running == 1)) { + if ((check_cpu_capacity(env->src_rq, sd)) && + (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) + return 1; + } + if ((capacity_of(env->src_cpu) < capacity_of(env->dst_cpu)) && + env->src_rq->cfs.h_nr_running == 1 && + cpu_overutilized(env->src_cpu) && + !cpu_overutilized(env->dst_cpu)) { + return 1; + } + return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2); } +static int active_load_balance_cpu_stop(void *data); + +static int should_we_balance(struct lb_env *env) +{ + struct sched_group *sg = env->sd->groups; + struct cpumask *sg_cpus, *sg_mask; + int cpu, balance_cpu = -1; + + /* + * In the newly idle case, we will allow all the cpu's + * to do the newly idle load balance. + */ + if (env->idle == CPU_NEWLY_IDLE) + return 1; + + sg_cpus = sched_group_cpus(sg); + sg_mask = sched_group_mask(sg); + /* Try to find first idle cpu */ + for_each_cpu_and(cpu, sg_cpus, env->cpus) { + if (!cpumask_test_cpu(cpu, sg_mask) || !idle_cpu(cpu)) + continue; + + balance_cpu = cpu; + break; + } + + if (balance_cpu == -1) + balance_cpu = group_balance_cpu(sg); + + /* + * First idle cpu or the first cpu(busiest) in this sched group + * is eligible for doing load balancing at this and above domains. + */ + return balance_cpu == env->dst_cpu; +} + /* * Check this_cpu to ensure it is balanced within domain. Attempt to move * tasks if there is an imbalance. */ static int load_balance(int this_cpu, struct rq *this_rq, struct sched_domain *sd, enum cpu_idle_type idle, - int *balance) + int *continue_balancing) { int ld_moved, cur_ld_moved, active_balance = 0; + struct sched_domain *sd_parent = sd->parent; struct sched_group *group; - struct rq *busiest = NULL; + struct rq *busiest; unsigned long flags; - struct cpumask *cpus = __get_cpu_var(load_balance_mask); + //struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); + struct cpumask *cpus = this_cpu_ptr(load_balance_mask); struct lb_env env = { - .sd = sd, - .dst_cpu = this_cpu, - .dst_rq = this_rq, - .dst_grpmask = sched_group_cpus(sd->groups), - .idle = idle, - .busiest_nr_running = 0, - .busiest_grp_capacity = 0, - .loop_break = sched_nr_migrate_break, - .cpus = cpus, - .flags = 0, - .loop = 0, + .sd = sd, + .dst_cpu = this_cpu, + .dst_rq = this_rq, + .dst_grpmask = sched_group_cpus(sd->groups), + .idle = idle, + .loop_break = sched_nr_migrate_break, + .cpus = cpus, + .fbq_type = all, + .tasks = LIST_HEAD_INIT(env.tasks), }; /* @@ -7602,15 +7589,15 @@ static int load_balance(int this_cpu, struct rq *this_rq, cpumask_copy(cpus, cpu_active_mask); - per_cpu(dbs_boost_load_moved, this_cpu) = 0; schedstat_inc(sd, lb_count[idle]); redo: - group = find_busiest_group(&env, balance); - - if (*balance == 0) + if (!should_we_balance(&env)) { + *continue_balancing = 0; goto out_balanced; + } + group = find_busiest_group(&env); if (!group) { schedstat_inc(sd, lb_nobusyg[idle]); goto out_balanced; @@ -7626,6 +7613,9 @@ static int load_balance(int this_cpu, struct rq *this_rq, schedstat_add(sd, lb_imbalance[idle], env.imbalance); + env.src_cpu = busiest->cpu; + env.src_rq = busiest; + ld_moved = 0; if (busiest->nr_running > 1) { /* @@ -7635,37 +7625,63 @@ static int load_balance(int this_cpu, struct rq *this_rq, * correctly treated as an imbalance. */ env.flags |= LBF_ALL_PINNED; - env.src_cpu = busiest->cpu; - env.src_rq = busiest; env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); - update_h_load(env.src_cpu); more_balance: - local_irq_save(flags); - double_rq_lock(env.dst_rq, busiest); - - /* The world might have changed. Validate assumptions */ - if (busiest->nr_running <= 1) { - double_rq_unlock(env.dst_rq, busiest); - local_irq_restore(flags); - env.flags &= ~LBF_ALL_PINNED; - goto no_move; - } + raw_spin_lock_irqsave(&busiest->lock, flags); /* * cur_ld_moved - load moved in current iteration * ld_moved - cumulative load moved across iterations */ - cur_ld_moved = move_tasks(&env); - ld_moved += cur_ld_moved; - double_rq_unlock(env.dst_rq, busiest); - local_irq_restore(flags); + cur_ld_moved = detach_tasks(&env); + /* + * We want to potentially update env.src_cpu's OPP. + * + * Add a margin (same ~20% used for the tipping point) + * to our request to provide some head room for the remaining + * tasks. + */ + if (sched_energy_freq() && cur_ld_moved) { + unsigned long req_cap = + get_boosted_cpu_usage(env.src_cpu); + + req_cap = req_cap * capacity_margin + >> SCHED_CAPACITY_SHIFT; + cpufreq_sched_set_cap(env.src_cpu, req_cap); + } /* - * some other cpu did the load balance for us. + * We've detached some tasks from busiest_rq. Every + * task is masked "TASK_ON_RQ_MIGRATING", so we can safely + * unlock busiest->lock, and we are able to be sure + * that nobody can manipulate the tasks in parallel. + * See task_rq_lock() family for the details. */ - if (cur_ld_moved && env.dst_cpu != smp_processor_id()) - resched_cpu(env.dst_cpu); + + raw_spin_unlock(&busiest->lock); + + if (cur_ld_moved) { + attach_tasks(&env); + ld_moved += cur_ld_moved; + /* + * We want to potentially update env.dst_cpu's OPP. + * + * Add a margin (same ~20% used for the tipping point) + * to our request to provide some head room if p's + * utilization further increases. + */ + if (sched_energy_freq()) { + unsigned long req_cap = + get_boosted_cpu_usage(env.dst_cpu); + + req_cap = req_cap * capacity_margin + >> SCHED_CAPACITY_SHIFT; + cpufreq_sched_set_cap(env.dst_cpu, req_cap); + } + } + + local_irq_restore(flags); if (env.flags & LBF_NEED_BREAK) { env.flags &= ~LBF_NEED_BREAK; @@ -7691,17 +7707,17 @@ static int load_balance(int this_cpu, struct rq *this_rq, * moreover subsequent load balance cycles should correct the * excess load moved. */ - if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) { + if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) { + + /* Prevent to re-select dst_cpu via env's cpus */ + cpumask_clear_cpu(env.dst_cpu, env.cpus); env.dst_rq = cpu_rq(env.new_dst_cpu); env.dst_cpu = env.new_dst_cpu; - env.flags &= ~LBF_SOME_PINNED; + env.flags &= ~LBF_DST_PINNED; env.loop = 0; env.loop_break = sched_nr_migrate_break; - /* Prevent to re-select dst_cpu via env's cpus */ - cpumask_clear_cpu(env.dst_cpu, env.cpus); - /* * Go back to "more_balance" rather than "redo" since we * need to continue with same src_cpu. @@ -7709,6 +7725,16 @@ static int load_balance(int this_cpu, struct rq *this_rq, goto more_balance; } + /* + * We failed to reach balance because of affinity. + */ + if (sd_parent) { + int *group_imbalance = &sd_parent->groups->sgc->imbalance; + + if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) + *group_imbalance = 1; + } + /* All tasks on this runqueue were pinned by CPU affinity */ if (unlikely(env.flags & LBF_ALL_PINNED)) { cpumask_clear_cpu(cpu_of(busiest), cpus); @@ -7717,24 +7743,21 @@ static int load_balance(int this_cpu, struct rq *this_rq, env.loop_break = sched_nr_migrate_break; goto redo; } - goto out_balanced; + goto out_all_pinned; } } -no_move: if (!ld_moved) { - if (!(env.flags & LBF_HMP_ACTIVE_BALANCE)) - schedstat_inc(sd, lb_failed[idle]); - + schedstat_inc(sd, lb_failed[idle]); /* * Increment the failure counter only on periodic balance. * We do not want newidle balance, which can be very * frequent, pollute the failure counter causing * excessive cache_hot migrations and active balances. */ - if (idle != CPU_NEWLY_IDLE && - !(env.flags & LBF_HMP_ACTIVE_BALANCE)) - sd->nr_balance_failed++; + if (idle != CPU_NEWLY_IDLE) + if (env.src_grp_nr_running > 1) + sd->nr_balance_failed++; if (need_active_balance(&env)) { raw_spin_lock_irqsave(&busiest->lock, flags); @@ -7767,7 +7790,6 @@ static int load_balance(int this_cpu, struct rq *this_rq, stop_one_cpu_nowait(cpu_of(busiest), active_load_balance_cpu_stop, busiest, &busiest->active_balance_work); - ld_moved++; } /* @@ -7776,29 +7798,9 @@ static int load_balance(int this_cpu, struct rq *this_rq, */ sd->nr_balance_failed = sd->cache_nice_tries+1; } - } else { + } else sd->nr_balance_failed = 0; - if (per_cpu(dbs_boost_needed, this_cpu)) { - struct migration_notify_data mnd; - - mnd.src_cpu = cpu_of(busiest); - mnd.dest_cpu = this_cpu; - mnd.load = per_cpu(dbs_boost_load_moved, this_cpu); - if (mnd.load > 100) - mnd.load = 100; - atomic_notifier_call_chain(&migration_notifier_head, - 0, (void *)&mnd); - per_cpu(dbs_boost_needed, this_cpu) = false; - per_cpu(dbs_boost_load_moved, this_cpu) = 0; - - } - /* Assumes one 'busiest' cpu that we pulled tasks from */ - if (!same_freq_domain(this_cpu, cpu_of(busiest))) { - check_for_freq_change(this_rq); - check_for_freq_change(busiest); - } - } if (likely(!active_balance)) { /* We were unbalanced, so reset the balancing interval */ sd->balance_interval = sd->min_interval; @@ -7807,7 +7809,7 @@ static int load_balance(int this_cpu, struct rq *this_rq, * If we've begun active balancing, start to back off. This * case may not be covered by the all_pinned logic if there * is only 1 task on the busy runqueue (because we don't call - * move_tasks). + * detach_tasks). */ if (sd->balance_interval < sd->max_interval) sd->balance_interval *= 2; @@ -7816,6 +7818,23 @@ static int load_balance(int this_cpu, struct rq *this_rq, goto out; out_balanced: + /* + * We reach balance although we may have faced some affinity + * constraints. Clear the imbalance flag if it was set. + */ + if (sd_parent) { + int *group_imbalance = &sd_parent->groups->sgc->imbalance; + + if (*group_imbalance) + *group_imbalance = 0; + } + +out_all_pinned: + /* + * We reach balance because all tasks are pinned at this level so + * we can't migrate them. Let the imbalance flag set so parent level + * can try to migrate them. + */ schedstat_inc(sd, lb_balanced[idle]); sd->nr_balance_failed = 0; @@ -7829,97 +7848,140 @@ static int load_balance(int this_cpu, struct rq *this_rq, ld_moved = 0; out: - trace_sched_load_balance(this_cpu, idle, *balance, - group ? group->cpumask[0] : 0, - busiest ? busiest->nr_running : 0, - env.imbalance, env.flags, ld_moved, - sd->balance_interval); return ld_moved; } +static inline unsigned long +get_sd_balance_interval(struct sched_domain *sd, int cpu_busy) +{ + unsigned long interval = sd->balance_interval; + + if (cpu_busy) + interval *= sd->busy_factor; + + /* scale ms to jiffies */ + interval = msecs_to_jiffies(interval); + interval = clamp(interval, 1UL, max_load_balance_interval); + + return interval; +} + +static inline void +update_next_balance(struct sched_domain *sd, int cpu_busy, unsigned long *next_balance) +{ + unsigned long interval, next; + + interval = get_sd_balance_interval(sd, cpu_busy); + next = sd->last_balance + interval; + + if (time_after(*next_balance, next)) + *next_balance = next; +} + /* * idle_balance is called by schedule() if this_cpu is about to become * idle. Attempts to pull tasks from other CPUs. */ -void idle_balance(int this_cpu, struct rq *this_rq) +static int idle_balance(struct rq *this_rq) { + unsigned long next_balance = jiffies + HZ; + int this_cpu = this_rq->cpu; struct sched_domain *sd; int pulled_task = 0; - unsigned long next_balance = jiffies + HZ; - int i, cost; - int min_power = INT_MAX; - int balance_cpu = -1; - struct rq *balance_rq = NULL; + u64 curr_cost = 0; - this_rq->idle_stamp = this_rq->clock; + idle_enter_fair(this_rq); - if (this_rq->avg_idle < sysctl_sched_migration_cost) - return; + /* + * We must set idle_stamp _before_ calling idle_balance(), such that we + * measure the duration of idle_balance() as idle time. + */ + this_rq->idle_stamp = rq_clock(this_rq); - /* If this CPU is not the most power-efficient idle CPU in the - * lowest level domain, run load balance on behalf of that - * most power-efficient idle CPU. */ - rcu_read_lock(); - sd = rcu_dereference(per_cpu(sd_llc, this_cpu)); - if (sd && sysctl_sched_enable_power_aware) { - for_each_cpu(i, sched_domain_span(sd)) { - if (i == this_cpu || idle_cpu(i)) { - cost = power_cost_at_freq(i, 0); - if (cost < min_power) { - min_power = cost; - balance_cpu = i; - } - } - } - BUG_ON(balance_cpu == -1); + if ((!energy_aware() && (this_rq->avg_idle < sysctl_sched_migration_cost + || !this_rq->rd->overload)) || + (energy_aware() && !this_rq->rd->overutilized)) { + rcu_read_lock(); + sd = rcu_dereference_check_sched_domain(this_rq->sd); + if (sd) + update_next_balance(sd, 0, &next_balance); + rcu_read_unlock(); - } else { - balance_cpu = this_cpu; + goto out; } - rcu_read_unlock(); - balance_rq = cpu_rq(balance_cpu); - /* * Drop the rq->lock, but keep IRQ/preempt disabled. */ raw_spin_unlock(&this_rq->lock); - update_blocked_averages(balance_cpu); + update_blocked_averages(this_cpu); rcu_read_lock(); - for_each_domain(balance_cpu, sd) { - unsigned long interval; - int balance = 1; + for_each_domain(this_cpu, sd) { + int continue_balancing = 1; + u64 t0, domain_cost; if (!(sd->flags & SD_LOAD_BALANCE)) continue; + if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) { + update_next_balance(sd, 0, &next_balance); + break; + } + if (sd->flags & SD_BALANCE_NEWIDLE) { - /* If we've pulled tasks over stop searching: */ - pulled_task = load_balance(balance_cpu, balance_rq, - sd, CPU_NEWLY_IDLE, &balance); + t0 = sched_clock_cpu(this_cpu); + + pulled_task = load_balance(this_cpu, this_rq, + sd, CPU_NEWLY_IDLE, + &continue_balancing); + + domain_cost = sched_clock_cpu(this_cpu) - t0; + if (domain_cost > sd->max_newidle_lb_cost) + sd->max_newidle_lb_cost = domain_cost; + + curr_cost += domain_cost; } - interval = msecs_to_jiffies(sd->balance_interval); - if (time_after(next_balance, sd->last_balance + interval)) - next_balance = sd->last_balance + interval; - if (pulled_task) { - balance_rq->idle_stamp = 0; + update_next_balance(sd, 0, &next_balance); + + /* + * Stop searching for tasks to pull if there are + * now runnable tasks on this rq. + */ + if (pulled_task || this_rq->nr_running > 0) break; - } } rcu_read_unlock(); raw_spin_lock(&this_rq->lock); - if (balance_cpu == this_cpu && - (!pulled_task || time_after(jiffies, this_rq->next_balance))) { - /* - * We are going idle. next_balance may be set based on - * a busy processor. So reset next_balance. - */ + if (curr_cost > this_rq->max_idle_balance_cost) + this_rq->max_idle_balance_cost = curr_cost; + + /* + * While browsing the domains, we released the rq lock, a task could + * have been enqueued in the meantime. Since we're not going idle, + * pretend we pulled a task. + */ + if (this_rq->cfs.h_nr_running && !pulled_task) + pulled_task = 1; + +out: + /* Move the next balance forward */ + if (time_after(this_rq->next_balance, next_balance)) this_rq->next_balance = next_balance; + + /* Is there a task of a high priority class? */ + if (this_rq->nr_running != this_rq->cfs.h_nr_running) + pulled_task = -1; + + if (pulled_task) { + idle_exit_fair(this_rq); + this_rq->idle_stamp = 0; } + + return pulled_task; } /* @@ -7934,26 +7996,11 @@ static int active_load_balance_cpu_stop(void *data) int busiest_cpu = cpu_of(busiest_rq); int target_cpu = busiest_rq->push_cpu; struct rq *target_rq = cpu_rq(target_cpu); - struct sched_domain *sd = NULL; - struct task_struct *push_task; - struct lb_env env = { - .sd = sd, - .dst_cpu = target_cpu, - .dst_rq = target_rq, - .src_cpu = busiest_rq->cpu, - .src_rq = busiest_rq, - .idle = CPU_IDLE, - .busiest_nr_running = 0, - .busiest_grp_capacity = 0, - .flags = 0, - .loop = 0, - }; - bool moved = false; + struct sched_domain *sd; + struct task_struct *p = NULL; raw_spin_lock_irq(&busiest_rq->lock); - per_cpu(dbs_boost_load_moved, target_cpu) = 0; - /* make sure the requested cpu hasn't gone down in the meantime */ if (unlikely(busiest_cpu != smp_processor_id() || !busiest_rq->active_balance)) @@ -7970,21 +8017,6 @@ static int active_load_balance_cpu_stop(void *data) */ BUG_ON(busiest_rq == target_rq); - /* move a task from busiest_rq to target_rq */ - double_lock_balance(busiest_rq, target_rq); - - push_task = busiest_rq->push_task; - target_cpu = busiest_rq->push_cpu; - if (push_task) { - if (push_task->on_rq && push_task->state == TASK_RUNNING && - task_cpu(push_task) == busiest_cpu && - cpu_online(target_cpu)) { - move_task(push_task, &env); - moved = true; - } - goto out_unlock_balance; - } - /* Search for an sd spanning us and the target CPU. */ rcu_read_lock(); for_each_domain(target_cpu, sd) { @@ -7994,52 +8026,73 @@ static int active_load_balance_cpu_stop(void *data) } if (likely(sd)) { - env.sd = sd; + struct lb_env env = { + .sd = sd, + .dst_cpu = target_cpu, + .dst_rq = target_rq, + .src_cpu = busiest_rq->cpu, + .src_rq = busiest_rq, + .idle = CPU_IDLE, + }; + schedstat_inc(sd, alb_count); - if (move_one_task(&env)) { + p = detach_one_task(&env); + if (p) { schedstat_inc(sd, alb_pushed); - moved = true; - } else { - schedstat_inc(sd, alb_failed); + /* + * We want to potentially update env.src_cpu's OPP. + * + * Add a margin (same ~20% used for the tipping point) + * to our request to provide some head room for the + * remaining task. + */ + if (sched_energy_freq()) { + unsigned long req_cap = + get_boosted_cpu_usage(env.src_cpu); + + req_cap = req_cap * capacity_margin + >> SCHED_CAPACITY_SHIFT; + cpufreq_sched_set_cap(env.src_cpu, req_cap); + } } + else + schedstat_inc(sd, alb_failed); } rcu_read_unlock(); -out_unlock_balance: - double_unlock_balance(busiest_rq, target_rq); out_unlock: busiest_rq->active_balance = 0; - push_task = busiest_rq->push_task; - target_cpu = busiest_rq->push_cpu; - if (push_task) { - put_task_struct(push_task); - clear_reserved(target_cpu); - busiest_rq->push_task = NULL; - } - raw_spin_unlock_irq(&busiest_rq->lock); + raw_spin_unlock(&busiest_rq->lock); - if (moved && !same_freq_domain(busiest_cpu, target_cpu)) { - check_for_freq_change(busiest_rq); - check_for_freq_change(target_rq); - } + if (p) { + attach_one_task(target_rq, p); + /* + * We want to potentially update target_cpu's OPP. + * + * Add a margin (same ~20% used for the tipping point) + * to our request to provide some head room if p's utilization + * further increases. + */ + if (sched_energy_freq()) { + unsigned long req_cap = + get_boosted_cpu_usage(target_cpu); - if (per_cpu(dbs_boost_needed, target_cpu)) { - struct migration_notify_data mnd; + req_cap = req_cap * capacity_margin + >> SCHED_CAPACITY_SHIFT; + cpufreq_sched_set_cap(target_cpu, req_cap); + } + } - mnd.src_cpu = cpu_of(busiest_rq); - mnd.dest_cpu = target_cpu; - mnd.load = per_cpu(dbs_boost_load_moved, target_cpu); - if (mnd.load > 100) - mnd.load = 100; - atomic_notifier_call_chain(&migration_notifier_head, - 0, (void *)&mnd); + local_irq_enable(); - per_cpu(dbs_boost_needed, target_cpu) = false; - per_cpu(dbs_boost_load_moved, target_cpu) = 0; - } return 0; } +static inline int on_null_domain(struct rq *rq) +{ + return unlikely(!rcu_dereference_sched(rq->sd)); +} + #ifdef CONFIG_NO_HZ_COMMON /* * idle load balancing details @@ -8053,14 +8106,9 @@ static struct { unsigned long next_balance; /* in jiffy units */ } nohz ____cacheline_aligned; -static inline int find_new_ilb(int call_cpu, int type) +static inline int find_new_ilb(void) { - int ilb; - - if (sched_enable_hmp) - return find_new_hmp_ilb(call_cpu, type); - - ilb = cpumask_first(nohz.idle_cpus_mask); + int ilb = cpumask_first(nohz.idle_cpus_mask); if (ilb < nr_cpu_ids && idle_cpu(ilb)) return ilb; @@ -8073,13 +8121,13 @@ static inline int find_new_ilb(int call_cpu, int type) * nohz_load_balancer CPU (if there is one) otherwise fallback to any idle * CPU (if there is one). */ -static void nohz_balancer_kick(int cpu, int type) +static void nohz_balancer_kick(void) { int ilb_cpu; nohz.next_balance++; - ilb_cpu = find_new_ilb(cpu, type); + ilb_cpu = find_new_ilb(); if (ilb_cpu >= nr_cpu_ids) return; @@ -8099,8 +8147,13 @@ static void nohz_balancer_kick(int cpu, int type) static inline void nohz_balance_exit_idle(int cpu) { if (unlikely(test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))) { - cpumask_clear_cpu(cpu, nohz.idle_cpus_mask); - atomic_dec(&nohz.nr_cpus); + /* + * Completely isolated CPUs don't ever set, so we must test. + */ + if (likely(cpumask_test_cpu(cpu, nohz.idle_cpus_mask))) { + cpumask_clear_cpu(cpu, nohz.idle_cpus_mask); + atomic_dec(&nohz.nr_cpus); + } clear_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)); } } @@ -8111,14 +8164,13 @@ static inline void set_cpu_sd_state_busy(void) int cpu = smp_processor_id(); rcu_read_lock(); - sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); + sd = rcu_dereference(per_cpu(sd_busy, cpu)); if (!sd || !sd->nohz_idle) goto unlock; sd->nohz_idle = 0; - for (; sd; sd = sd->parent) - atomic_inc(&sd->groups->sgp->nr_busy_cpus); + atomic_inc(&sd->groups->sgc->nr_busy_cpus); unlock: rcu_read_unlock(); } @@ -8129,14 +8181,13 @@ void set_cpu_sd_state_idle(void) int cpu = smp_processor_id(); rcu_read_lock(); - sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); + sd = rcu_dereference(per_cpu(sd_busy, cpu)); if (!sd || sd->nohz_idle) goto unlock; sd->nohz_idle = 1; - for (; sd; sd = sd->parent) - atomic_dec(&sd->groups->sgp->nr_busy_cpus); + atomic_dec(&sd->groups->sgc->nr_busy_cpus); unlock: rcu_read_unlock(); } @@ -8156,12 +8207,18 @@ void nohz_balance_enter_idle(int cpu) if (test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu))) return; + /* + * If we're a completely isolated CPU, we don't play. + */ + if (on_null_domain(cpu_rq(cpu))) + return; + cpumask_set_cpu(cpu, nohz.idle_cpus_mask); atomic_inc(&nohz.nr_cpus); set_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)); } -static int __cpuinit sched_ilb_notifier(struct notifier_block *nfb, +static int sched_ilb_notifier(struct notifier_block *nfb, unsigned long action, void *hcpu) { switch (action & ~CPU_TASKS_FROZEN) { @@ -8191,49 +8248,67 @@ void update_max_interval(void) * * Balancing parameters are set up in init_sched_domains. */ -static void rebalance_domains(int cpu, enum cpu_idle_type idle) +static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle) { - int balance = 1; - struct rq *rq = cpu_rq(cpu); + int continue_balancing = 1; + int cpu = rq->cpu; unsigned long interval; struct sched_domain *sd; /* Earliest time when we have to do rebalance again */ unsigned long next_balance = jiffies + 60*HZ; int update_next_balance = 0; - int need_serialize; + int need_serialize, need_decay = 0; + u64 max_cost = 0; update_blocked_averages(cpu); rcu_read_lock(); for_each_domain(cpu, sd) { + /* + * Decay the newidle max times here because this is a regular + * visit to all the domains. Decay ~1% per second. + */ + if (time_after(jiffies, sd->next_decay_max_lb_cost)) { + sd->max_newidle_lb_cost = + (sd->max_newidle_lb_cost * 253) / 256; + sd->next_decay_max_lb_cost = jiffies + HZ; + need_decay = 1; + } + max_cost += sd->max_newidle_lb_cost; + if (!(sd->flags & SD_LOAD_BALANCE)) continue; - interval = sd->balance_interval; - if (idle != CPU_IDLE) - interval *= sd->busy_factor; + /* + * Stop the load balance at this level. There is another + * CPU in our sched group which is doing load balancing more + * actively. + */ + if (!continue_balancing) { + if (need_decay) + continue; + break; + } - /* scale ms to jiffies */ - interval = msecs_to_jiffies(interval); - interval = clamp(interval, 1UL, max_load_balance_interval); + interval = get_sd_balance_interval(sd, idle != CPU_IDLE); need_serialize = sd->flags & SD_SERIALIZE; - if (need_serialize) { if (!spin_trylock(&balancing)) goto out; } if (time_after_eq(jiffies, sd->last_balance + interval)) { - if (load_balance(cpu, rq, sd, idle, &balance)) { + if (load_balance(cpu, rq, sd, idle, &continue_balancing)) { /* - * The LBF_SOME_PINNED logic could have changed + * The LBF_DST_PINNED logic could have changed * env->dst_cpu, so we can't know our idle * state even if we migrated tasks. Update it. */ idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE; } sd->last_balance = jiffies; + interval = get_sd_balance_interval(sd, idle != CPU_IDLE); } if (need_serialize) spin_unlock(&balancing); @@ -8242,14 +8317,14 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle) next_balance = sd->last_balance + interval; update_next_balance = 1; } - + } + if (need_decay) { /* - * Stop the load balance at this level. There is another - * CPU in our sched group which is doing load balancing more - * actively. + * Ensure the rq-wide value also decays but keep it at a + * reasonable floor to avoid funnies with rq->avg_idle. */ - if (!balance) - break; + rq->max_idle_balance_cost = + max((u64)sysctl_sched_migration_cost, max_cost); } rcu_read_unlock(); @@ -8263,49 +8338,21 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle) } #ifdef CONFIG_NO_HZ_COMMON - -static int select_lowest_power_cpu(struct cpumask *cpus) -{ - int i, cost; - int lowest_power_cpu = -1; - int lowest_power = INT_MAX; - - if (sysctl_sched_enable_power_aware) { - for_each_cpu(i, cpus) { - cost = power_cost_at_freq(i, 0); - if (cost < lowest_power) { - lowest_power_cpu = i; - lowest_power = cost; - } - } - BUG_ON(lowest_power_cpu == -1); - return lowest_power_cpu; - } else { - return cpumask_first(cpus); - } -} - /* * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the * rebalancing for all the cpus for whom scheduler ticks are stopped. */ -static void nohz_idle_balance(int this_cpu, enum cpu_idle_type idle) +static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { - struct rq *this_rq = cpu_rq(this_cpu); + int this_cpu = this_rq->cpu; struct rq *rq; int balance_cpu; - struct cpumask cpus_to_balance; if (idle != CPU_IDLE || !test_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu))) goto end; - cpumask_copy(&cpus_to_balance, nohz.idle_cpus_mask); - - while (!cpumask_empty(&cpus_to_balance)) { - balance_cpu = select_lowest_power_cpu(&cpus_to_balance); - - cpumask_clear_cpu(balance_cpu, &cpus_to_balance); + for_each_cpu(balance_cpu, nohz.idle_cpus_mask) { if (balance_cpu == this_cpu || !idle_cpu(balance_cpu)) continue; @@ -8319,12 +8366,17 @@ static void nohz_idle_balance(int this_cpu, enum cpu_idle_type idle) rq = cpu_rq(balance_cpu); - raw_spin_lock_irq(&rq->lock); - update_rq_clock(rq); - update_idle_cpu_load(rq); - raw_spin_unlock_irq(&rq->lock); - - rebalance_domains(balance_cpu, CPU_IDLE); + /* + * If time for next balance is due, + * do the balance. + */ + if (time_after_eq(jiffies, rq->next_balance)) { + raw_spin_lock_irq(&rq->lock); + update_rq_clock(rq); + update_idle_cpu_load(rq); + raw_spin_unlock_irq(&rq->lock); + rebalance_domains(rq, CPU_IDLE); + } if (time_after(this_rq->next_balance, rq->next_balance)) this_rq->next_balance = rq->next_balance; @@ -8334,88 +8386,27 @@ static void nohz_idle_balance(int this_cpu, enum cpu_idle_type idle) clear_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu)); } -#ifdef CONFIG_SCHED_HMP -static inline int _nohz_kick_needed_hmp(struct rq *rq, int cpu, int *type) -{ - struct sched_domain *sd; - int i; - - if (rq->mostly_idle_freq && rq->cur_freq < rq->mostly_idle_freq - && rq->max_freq > rq->mostly_idle_freq) - return 0; - - if (rq->nr_running >= 2 && - (rq->nr_running - rq->hmp_stats.nr_small_tasks >= 2 || - rq->nr_running > rq->mostly_idle_nr_run || - cpu_load(cpu) > rq->mostly_idle_load)) { - - if (rq->capacity == max_capacity) - return 1; - - rcu_read_lock(); - sd = rcu_dereference_check_sched_domain(rq->sd); - if (!sd) { - rcu_read_unlock(); - return 0; - } - - for_each_cpu(i, sched_domain_span(sd)) { - if (cpu_load(i) < sched_spill_load) { - /* Change the kick type to limit to CPUs that - * are of equal or lower capacity. - */ - *type = NOHZ_KICK_RESTRICT; - break; - } - } - rcu_read_unlock(); - return 1; - } - - return 0; -} -#else -static inline int _nohz_kick_needed_hmp(struct rq *rq, int cpu, int *type) -{ - return 0; -} -#endif - -static inline int _nohz_kick_needed(struct rq *rq, int cpu, int *type) -{ - unsigned long now = jiffies; - - if (sched_enable_hmp) - return _nohz_kick_needed_hmp(rq, cpu, type); - - /* - * None are in tickless mode and hence no need for NOHZ idle load - * balancing. - */ - if (likely(!atomic_read(&nohz.nr_cpus))) - return 0; - - if (time_before(now, nohz.next_balance)) - return 0; - - return (rq->nr_running >= 2); -} - /* * Current heuristic for kicking the idle load balancer in the presence - * of an idle cpu is the system. + * of an idle cpu in the system. * - This rq has more than one task. - * - At any scheduler domain level, this cpu's scheduler group has multiple - * busy cpu's exceeding the group's power. + * - This rq has at least one CFS task and the capacity of the CPU is + * significantly reduced because of RT tasks or IRQs. + * - At parent of LLC scheduler domain level, this cpu's scheduler group has + * multiple busy cpu. * - For SD_ASYM_PACKING, if the lower numbered cpu's in the scheduler * domain span are idle. */ -static inline int nohz_kick_needed(struct rq *rq, int cpu, int *type) +static inline bool nohz_kick_needed(struct rq *rq) { + unsigned long now = jiffies; struct sched_domain *sd; + struct sched_group_capacity *sgc; + int nr_busy, cpu = rq->cpu; + bool kick = false; - if (unlikely(idle_cpu(cpu))) - return 0; + if (unlikely(rq->idle_balance)) + return false; /* * We may be recently in ticked or tickless idle mode. At the first @@ -8424,38 +8415,55 @@ static inline int nohz_kick_needed(struct rq *rq, int cpu, int *type) set_cpu_sd_state_busy(); nohz_balance_exit_idle(cpu); - if (_nohz_kick_needed(rq, cpu, type)) - goto need_kick; + /* + * None are in tickless mode and hence no need for NOHZ idle load + * balancing. + */ + if (likely(!atomic_read(&nohz.nr_cpus))) + return false; + + if (time_before(now, nohz.next_balance)) + return false; + + if (rq->nr_running >= 2 && + (!energy_aware() || cpu_overutilized(cpu))) + return true; rcu_read_lock(); - for_each_domain(cpu, sd) { - struct sched_group *sg = sd->groups; - struct sched_group_power *sgp = sg->sgp; - int nr_busy = atomic_read(&sgp->nr_busy_cpus); + sd = rcu_dereference(per_cpu(sd_busy, cpu)); + if (sd && !energy_aware()) { + sgc = sd->groups->sgc; + nr_busy = atomic_read(&sgc->nr_busy_cpus); + + if (nr_busy > 1) { + kick = true; + goto unlock; + } -#ifndef CONFIG_SCHED_HMP - if (sd->flags & SD_SHARE_PKG_RESOURCES && nr_busy > 1) - goto need_kick_unlock; -#endif + } - if (sd->flags & SD_ASYM_PACKING && nr_busy != sg->group_weight - && (cpumask_first_and(nohz.idle_cpus_mask, - sched_domain_span(sd)) < cpu)) - goto need_kick_unlock; + sd = rcu_dereference(rq->sd); + if (sd) { + if ((rq->cfs.h_nr_running >= 1) && + check_cpu_capacity(rq, sd)) { + kick = true; + goto unlock; + } + } - if (!(sd->flags & (SD_SHARE_PKG_RESOURCES | SD_ASYM_PACKING))) - break; + sd = rcu_dereference(per_cpu(sd_asym, cpu)); + if (sd && (cpumask_first_and(nohz.idle_cpus_mask, + sched_domain_span(sd)) < cpu)) { + kick = true; + goto unlock; } - rcu_read_unlock(); - return 0; -need_kick_unlock: +unlock: rcu_read_unlock(); -need_kick: - return 1; + return kick; } #else -static void nohz_idle_balance(int this_cpu, enum cpu_idle_type idle) { } +static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { } #endif /* @@ -8464,40 +8472,36 @@ static void nohz_idle_balance(int this_cpu, enum cpu_idle_type idle) { } */ static void run_rebalance_domains(struct softirq_action *h) { - int this_cpu = smp_processor_id(); - struct rq *this_rq = cpu_rq(this_cpu); + struct rq *this_rq = this_rq(); enum cpu_idle_type idle = this_rq->idle_balance ? CPU_IDLE : CPU_NOT_IDLE; - rebalance_domains(this_cpu, idle); - /* * If this cpu has a pending nohz_balance_kick, then do the * balancing on behalf of the other idle cpus whose ticks are - * stopped. + * stopped. Do nohz_idle_balance *before* rebalance_domains to + * give the idle cpus a chance to load balance. Else we may + * load balance only within the local sched_domain hierarchy + * and abort nohz_idle_balance altogether if we pull some load. */ - nohz_idle_balance(this_cpu, idle); -} - -static inline int on_null_domain(int cpu) -{ - return !rcu_dereference_sched(cpu_rq(cpu)->sd); + nohz_idle_balance(this_rq, idle); + rebalance_domains(this_rq, idle); } /* * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing. */ -void trigger_load_balance(struct rq *rq, int cpu) +void trigger_load_balance(struct rq *rq) { - int type = NOHZ_KICK_ANY; - /* Don't need to rebalance while attached to NULL domain */ - if (time_after_eq(jiffies, rq->next_balance) && - likely(!on_null_domain(cpu))) + if (unlikely(on_null_domain(rq))) + return; + + if (time_after_eq(jiffies, rq->next_balance)) raise_softirq(SCHED_SOFTIRQ); #ifdef CONFIG_NO_HZ_COMMON - if (nohz_kick_needed(rq, cpu, &type) && likely(!on_null_domain(cpu))) - nohz_balancer_kick(cpu, type); + if (nohz_kick_needed(rq)) + nohz_balancer_kick(); #endif } @@ -8531,10 +8535,34 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) entity_tick(cfs_rq, se, queued); } - if (sched_feat_numa(NUMA)) + if (numabalancing_enabled) task_tick_numa(rq, curr); update_rq_runnable_avg(rq, 1); + + if (!rq->rd->overutilized && cpu_overutilized(task_cpu(curr))) + rq->rd->overutilized = true; + + rq->misfit_task = !task_fits_capacity(curr, rq->cpu); + + /* + * To make free room for a task that is building up its "real" + * utilization and to harm its performance the least, request a + * jump to max OPP as soon as get_cpu_usage() crosses the UP + * threshold. The UP threshold is built relative to the current + * capacity (OPP), by using same margin used to tell if a cpu + * is overutilized (capacity_margin). + */ + if (sched_energy_freq()) { + int cpu = cpu_of(rq); + unsigned long capacity_orig = capacity_orig_of(cpu); + unsigned long capacity_curr = capacity_curr_of(cpu); + + if (capacity_curr < capacity_orig && + (capacity_curr * SCHED_LOAD_SCALE) < + (get_cpu_usage(cpu) * capacity_margin)) + cpufreq_sched_set_cap(cpu, capacity_orig); + } } /* @@ -8579,7 +8607,7 @@ static void task_fork_fair(struct task_struct *p) * 'current' within the tree based on its new key value. */ swap(curr->vruntime, se->vruntime); - resched_task(rq->curr); + resched_curr(rq); } se->vruntime -= cfs_rq->min_vruntime; @@ -8594,7 +8622,7 @@ static void task_fork_fair(struct task_struct *p) static void prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio) { - if (!p->se.on_rq) + if (!task_on_rq_queued(p)) return; /* @@ -8604,7 +8632,7 @@ prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio) */ if (rq->curr == p) { if (p->prio > oldprio) - resched_task(rq->curr); + resched_curr(rq); } else check_preempt_curr(rq, p, 0); } @@ -8619,11 +8647,11 @@ static void switched_from_fair(struct rq *rq, struct task_struct *p) * switched back to the fair class the enqueue_entity(.flags=0) will * do the right thing. * - * If it's on_rq, then the dequeue_entity(.flags=0) will already - * have normalized the vruntime, if it's !on_rq, then only when + * If it's queued, then the dequeue_entity(.flags=0) will already + * have normalized the vruntime, if it's !queued, then only when * the task is sleeping will it still have non-normalized vruntime. */ - if (!p->on_rq && p->state != TASK_RUNNING) { + if (!task_on_rq_queued(p) && p->state != TASK_RUNNING) { /* * Fix up our vruntime so that the current sleep doesn't * cause 'unlimited' sleep bonus. @@ -8632,17 +8660,17 @@ static void switched_from_fair(struct rq *rq, struct task_struct *p) se->vruntime -= cfs_rq->min_vruntime; } -#if defined(CONFIG_FAIR_GROUP_SCHED) && defined(CONFIG_SMP) +#ifdef CONFIG_SMP /* * Remove our load from contribution when we leave sched_fair * and ensure we don't carry in an old decay_count if we * switch back. */ - if (p->se.avg.decay_count) { - struct cfs_rq *cfs_rq = cfs_rq_of(&p->se); - __synchronize_entity_decay(&p->se); - subtract_blocked_load_contrib(cfs_rq, - p->se.avg.load_avg_contrib); + if (se->avg.decay_count) { + __synchronize_entity_decay(se); + subtract_blocked_load_contrib(cfs_rq, se->avg.load_avg_contrib); + subtract_utilization_blocked_contrib(cfs_rq, + se->avg.utilization_avg_contrib); } #endif } @@ -8652,7 +8680,15 @@ static void switched_from_fair(struct rq *rq, struct task_struct *p) */ static void switched_to_fair(struct rq *rq, struct task_struct *p) { - if (!p->se.on_rq) +#ifdef CONFIG_FAIR_GROUP_SCHED + struct sched_entity *se = &p->se; + /* + * Since the real-depth could have been changed (only FAIR + * class maintain depth value), reset depth properly. + */ + se->depth = se->parent ? se->parent->depth + 1 : 0; +#endif + if (!task_on_rq_queued(p)) return; /* @@ -8661,7 +8697,7 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p) * if we can still preempt the current task. */ if (rq->curr == p) - resched_task(rq->curr); + resched_curr(rq); else check_preempt_curr(rq, p, 0); } @@ -8691,16 +8727,19 @@ void init_cfs_rq(struct cfs_rq *cfs_rq) #ifndef CONFIG_64BIT cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; #endif -#if defined(CONFIG_FAIR_GROUP_SCHED) && defined(CONFIG_SMP) +#ifdef CONFIG_SMP atomic64_set(&cfs_rq->decay_counter, 1); - atomic64_set(&cfs_rq->removed_load, 0); + atomic_long_set(&cfs_rq->removed_load, 0); + atomic_long_set(&cfs_rq->removed_utilization, 0); #endif } #ifdef CONFIG_FAIR_GROUP_SCHED -static void task_move_group_fair(struct task_struct *p, int on_rq) +static void task_move_group_fair(struct task_struct *p, int queued) { + struct sched_entity *se = &p->se; struct cfs_rq *cfs_rq; + /* * If the task was not on the rq at the time of this cgroup movement * it must have been asleep, sleeping tasks keep their ->vruntime @@ -8715,7 +8754,7 @@ static void task_move_group_fair(struct task_struct *p, int on_rq) * fair sleeper stuff for the first placement, but who cares. */ /* - * When !on_rq, vruntime of the task has usually NOT been normalized. + * When !queued, vruntime of the task has usually NOT been normalized. * But there are some cases where it has already been normalized: * * - Moving a forked child which is waiting for being woken up by @@ -8726,23 +8765,26 @@ static void task_move_group_fair(struct task_struct *p, int on_rq) * To prevent boost or penalty in the new cfs_rq caused by delta * min_vruntime between the two cfs_rqs, we skip vruntime adjustment. */ - if (!on_rq && (!p->se.sum_exec_runtime || p->state == TASK_WAKING)) - on_rq = 1; + if (!queued && (!se->sum_exec_runtime || p->state == TASK_WAKING)) + queued = 1; - if (!on_rq) - p->se.vruntime -= cfs_rq_of(&p->se)->min_vruntime; + if (!queued) + se->vruntime -= cfs_rq_of(se)->min_vruntime; set_task_rq(p, task_cpu(p)); - if (!on_rq) { - cfs_rq = cfs_rq_of(&p->se); - p->se.vruntime += cfs_rq->min_vruntime; + se->depth = se->parent ? se->parent->depth + 1 : 0; + if (!queued) { + cfs_rq = cfs_rq_of(se); + se->vruntime += cfs_rq->min_vruntime; #ifdef CONFIG_SMP /* * migrate_task_rq_fair() will have removed our previous * contribution, but we must synchronize for ongoing future * decay. */ - p->se.avg.decay_count = atomic64_read(&cfs_rq->decay_counter); - cfs_rq->blocked_load_avg += p->se.avg.load_avg_contrib; + se->avg.decay_count = atomic64_read(&cfs_rq->decay_counter); + cfs_rq->blocked_load_avg += se->avg.load_avg_contrib; + cfs_rq->utilization_blocked_avg += + se->avg.utilization_avg_contrib; #endif } } @@ -8838,10 +8880,13 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, if (!se) return; - if (!parent) + if (!parent) { se->cfs_rq = &rq->cfs; - else + se->depth = 0; + } else { se->cfs_rq = parent->my_q; + se->depth = parent->depth + 1; + } se->my_q = cfs_rq; /* guarantee group entities always have weight */ @@ -8876,6 +8921,9 @@ int sched_group_set_shares(struct task_group *tg, unsigned long shares) se = tg->se[i]; /* Propagate contribution to hierarchy */ raw_spin_lock_irqsave(&rq->lock, flags); + + /* Possible calls to update_curr() need rq clock */ + update_rq_clock(rq); for_each_sched_entity(se) update_cfs_shares(group_cfs_rq(se)); raw_spin_unlock_irqrestore(&rq->lock, flags); @@ -8931,9 +8979,8 @@ const struct sched_class fair_sched_class = { #ifdef CONFIG_SMP .select_task_rq = select_task_rq_fair, -#ifdef CONFIG_FAIR_GROUP_SCHED .migrate_task_rq = migrate_task_rq_fair, -#endif + .rq_online = rq_online_fair, .rq_offline = rq_offline_fair, @@ -8950,13 +8997,11 @@ const struct sched_class fair_sched_class = { .get_rr_interval = get_rr_interval_fair, + .update_curr = update_curr_fair, + #ifdef CONFIG_FAIR_GROUP_SCHED .task_move_group = task_move_group_fair, #endif -#ifdef CONFIG_SCHED_HMP - .inc_hmp_sched_stats = inc_hmp_sched_stats_fair, - .dec_hmp_sched_stats = dec_hmp_sched_stats_fair, -#endif }; #ifdef CONFIG_SCHED_DEBUG diff --git a/kernel/sched/features.h b/kernel/sched/features.h index cbb00d524aef4..eb53b51f374dd 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -36,19 +36,14 @@ SCHED_FEAT(CACHE_HOT_BUDDY, true) */ SCHED_FEAT(WAKEUP_PREEMPTION, true) -/* - * Use arch dependent cpu power functions - */ -SCHED_FEAT(ARCH_POWER, false) - SCHED_FEAT(HRTICK, false) SCHED_FEAT(DOUBLE_TICK, false) SCHED_FEAT(LB_BIAS, true) /* - * Decrement CPU power based on time not spent running tasks + * Decrement CPU capacity based on time not spent running tasks */ -SCHED_FEAT(NONTASK_POWER, true) +SCHED_FEAT(NONTASK_CAPACITY, true) /* * Queue remote wakeups on the target CPU and process them @@ -63,12 +58,29 @@ SCHED_FEAT(LB_MIN, false) /* * Apply the automatic NUMA scheduling policy. Enabled automatically * at runtime if running on a NUMA machine. Can be controlled via - * numa_balancing=. Allow PTE scanning to be forced on UMA machines - * for debugging the core machinery. + * numa_balancing= */ #ifdef CONFIG_NUMA_BALANCING SCHED_FEAT(NUMA, false) -SCHED_FEAT(NUMA_FORCE, false) + +/* + * NUMA_FAVOUR_HIGHER will favor moving tasks towards nodes where a + * higher number of hinting faults are recorded during active load + * balancing. + */ +SCHED_FEAT(NUMA_FAVOUR_HIGHER, true) + +/* + * NUMA_RESIST_LOWER will resist moving tasks towards nodes where a + * lower number of hinting faults have been recorded. As this has + * the potential to prevent a task ever migrating to a new node + * due to CPU overload it is disabled by default. + */ +SCHED_FEAT(NUMA_RESIST_LOWER, false) #endif -SCHED_FEAT(FORCE_CPU_THROTTLING_IMMINENT, false) +/* + * Energy aware scheduling. Use platform energy model to guide scheduling + * decisions optimizing for energy efficiency. + */ +SCHED_FEAT(ENERGY_AWARE, true) diff --git a/kernel/sched/idle_task.c b/kernel/sched/idle_task.c index d23d2964844e3..c65dac8c97cdd 100644 --- a/kernel/sched/idle_task.c +++ b/kernel/sched/idle_task.c @@ -9,37 +9,26 @@ #ifdef CONFIG_SMP static int -select_task_rq_idle(struct task_struct *p, int sd_flag, int flags) +select_task_rq_idle(struct task_struct *p, int cpu, int sd_flag, int flags) { return task_cpu(p); /* IDLE tasks as never migrated */ } - -static void pre_schedule_idle(struct rq *rq, struct task_struct *prev) -{ - idle_exit_fair(rq); - rq_last_tick_reset(rq); -} - -static void post_schedule_idle(struct rq *rq) -{ - idle_enter_fair(rq); -} #endif /* CONFIG_SMP */ + /* * Idle tasks are unconditionally rescheduled: */ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int flags) { - resched_task(rq->idle); + resched_curr(rq); } -static struct task_struct *pick_next_task_idle(struct rq *rq) +static struct task_struct * +pick_next_task_idle(struct rq *rq, struct task_struct *prev) { + put_prev_task(rq, prev); + schedstat_inc(rq, sched_goidle); -#ifdef CONFIG_SMP - /* Trigger the post schedule to do an idle_enter for CFS */ - rq->post_schedule = 1; -#endif return rq->idle; } @@ -58,6 +47,8 @@ dequeue_task_idle(struct rq *rq, struct task_struct *p, int flags) static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) { + idle_exit_fair(rq); + rq_last_tick_reset(rq); } static void task_tick_idle(struct rq *rq, struct task_struct *curr, int queued) @@ -84,20 +75,10 @@ static unsigned int get_rr_interval_idle(struct rq *rq, struct task_struct *task return 0; } -#ifdef CONFIG_SCHED_HMP - -static void -inc_hmp_sched_stats_idle(struct rq *rq, struct task_struct *p) +static void update_curr_idle(struct rq *rq) { } -static void -dec_hmp_sched_stats_idle(struct rq *rq, struct task_struct *p) -{ -} - -#endif - /* * Simple, special scheduling class for the per-CPU idle tasks: */ @@ -115,8 +96,6 @@ const struct sched_class idle_sched_class = { #ifdef CONFIG_SMP .select_task_rq = select_task_rq_idle, - .pre_schedule = pre_schedule_idle, - .post_schedule = post_schedule_idle, #endif .set_curr_task = set_curr_task_idle, @@ -126,8 +105,5 @@ const struct sched_class idle_sched_class = { .prio_changed = prio_changed_idle, .switched_to = switched_to_idle, -#ifdef CONFIG_SCHED_HMP - .inc_hmp_sched_stats = inc_hmp_sched_stats_idle, - .dec_hmp_sched_stats = dec_hmp_sched_stats_idle, -#endif + .update_curr = update_curr_idle, }; diff --git a/kernel/sched/proc.c b/kernel/sched/proc.c new file mode 100644 index 0000000000000..8ecd552fe4f22 --- /dev/null +++ b/kernel/sched/proc.c @@ -0,0 +1,584 @@ +/* + * kernel/sched/proc.c + * + * Kernel load calculations, forked from sched/core.c + */ + +#include + +#include "sched.h" + +/* + * Global load-average calculations + * + * We take a distributed and async approach to calculating the global load-avg + * in order to minimize overhead. + * + * The global load average is an exponentially decaying average of nr_running + + * nr_uninterruptible. + * + * Once every LOAD_FREQ: + * + * nr_active = 0; + * for_each_possible_cpu(cpu) + * nr_active += cpu_of(cpu)->nr_running + cpu_of(cpu)->nr_uninterruptible; + * + * avenrun[n] = avenrun[0] * exp_n + nr_active * (1 - exp_n) + * + * Due to a number of reasons the above turns in the mess below: + * + * - for_each_possible_cpu() is prohibitively expensive on machines with + * serious number of cpus, therefore we need to take a distributed approach + * to calculating nr_active. + * + * \Sum_i x_i(t) = \Sum_i x_i(t) - x_i(t_0) | x_i(t_0) := 0 + * = \Sum_i { \Sum_j=1 x_i(t_j) - x_i(t_j-1) } + * + * So assuming nr_active := 0 when we start out -- true per definition, we + * can simply take per-cpu deltas and fold those into a global accumulate + * to obtain the same result. See calc_load_fold_active(). + * + * Furthermore, in order to avoid synchronizing all per-cpu delta folding + * across the machine, we assume 10 ticks is sufficient time for every + * cpu to have completed this task. + * + * This places an upper-bound on the IRQ-off latency of the machine. Then + * again, being late doesn't loose the delta, just wrecks the sample. + * + * - cpu_rq()->nr_uninterruptible isn't accurately tracked per-cpu because + * this would add another cross-cpu cacheline miss and atomic operation + * to the wakeup path. Instead we increment on whatever cpu the task ran + * when it went into uninterruptible state and decrement on whatever cpu + * did the wakeup. This means that only the sum of nr_uninterruptible over + * all cpus yields the correct result. + * + * This covers the NO_HZ=n code, for extra head-aches, see the comment below. + */ + +/* Variables and functions for calc_load */ +atomic_long_t calc_load_tasks; +unsigned long calc_load_update; +unsigned long avenrun[3]; +EXPORT_SYMBOL(avenrun); /* should be removed */ + +/** + * get_avenrun - get the load average array + * @loads: pointer to dest load array + * @offset: offset to add + * @shift: shift count to shift the result left + * + * These values are estimates at best, so no need for locking. + */ +void get_avenrun(unsigned long *loads, unsigned long offset, int shift) +{ + loads[0] = (avenrun[0] + offset) << shift; + loads[1] = (avenrun[1] + offset) << shift; + loads[2] = (avenrun[2] + offset) << shift; +} + +long calc_load_fold_active(struct rq *this_rq) +{ + long nr_active, delta = 0; + + nr_active = this_rq->nr_running; + nr_active += (long) this_rq->nr_uninterruptible; + + if (nr_active != this_rq->calc_load_active) { + delta = nr_active - this_rq->calc_load_active; + this_rq->calc_load_active = nr_active; + } + + return delta; +} + +/* + * a1 = a0 * e + a * (1 - e) + */ +static unsigned long +calc_load(unsigned long load, unsigned long exp, unsigned long active) +{ + load *= exp; + load += active * (FIXED_1 - exp); + load += 1UL << (FSHIFT - 1); + return load >> FSHIFT; +} + +#ifdef CONFIG_NO_HZ_COMMON +/* + * Handle NO_HZ for the global load-average. + * + * Since the above described distributed algorithm to compute the global + * load-average relies on per-cpu sampling from the tick, it is affected by + * NO_HZ. + * + * The basic idea is to fold the nr_active delta into a global idle-delta upon + * entering NO_HZ state such that we can include this as an 'extra' cpu delta + * when we read the global state. + * + * Obviously reality has to ruin such a delightfully simple scheme: + * + * - When we go NO_HZ idle during the window, we can negate our sample + * contribution, causing under-accounting. + * + * We avoid this by keeping two idle-delta counters and flipping them + * when the window starts, thus separating old and new NO_HZ load. + * + * The only trick is the slight shift in index flip for read vs write. + * + * 0s 5s 10s 15s + * +10 +10 +10 +10 + * |-|-----------|-|-----------|-|-----------|-| + * r:0 0 1 1 0 0 1 1 0 + * w:0 1 1 0 0 1 1 0 0 + * + * This ensures we'll fold the old idle contribution in this window while + * accumlating the new one. + * + * - When we wake up from NO_HZ idle during the window, we push up our + * contribution, since we effectively move our sample point to a known + * busy state. + * + * This is solved by pushing the window forward, and thus skipping the + * sample, for this cpu (effectively using the idle-delta for this cpu which + * was in effect at the time the window opened). This also solves the issue + * of having to deal with a cpu having been in NOHZ idle for multiple + * LOAD_FREQ intervals. + * + * When making the ILB scale, we should try to pull this in as well. + */ +static atomic_long_t calc_load_idle[2]; +static int calc_load_idx; + +static inline int calc_load_write_idx(void) +{ + int idx = calc_load_idx; + + /* + * See calc_global_nohz(), if we observe the new index, we also + * need to observe the new update time. + */ + smp_rmb(); + + /* + * If the folding window started, make sure we start writing in the + * next idle-delta. + */ + if (!time_before(jiffies, calc_load_update)) + idx++; + + return idx & 1; +} + +static inline int calc_load_read_idx(void) +{ + return calc_load_idx & 1; +} + +void calc_load_enter_idle(void) +{ + struct rq *this_rq = this_rq(); + long delta; + + /* + * We're going into NOHZ mode, if there's any pending delta, fold it + * into the pending idle delta. + */ + delta = calc_load_fold_active(this_rq); + if (delta) { + int idx = calc_load_write_idx(); + atomic_long_add(delta, &calc_load_idle[idx]); + } +} + +void calc_load_exit_idle(void) +{ + struct rq *this_rq = this_rq(); + + /* + * If we're still before the sample window, we're done. + */ + if (time_before(jiffies, this_rq->calc_load_update)) + return; + + /* + * We woke inside or after the sample window, this means we're already + * accounted through the nohz accounting, so skip the entire deal and + * sync up for the next window. + */ + this_rq->calc_load_update = calc_load_update; + if (time_before(jiffies, this_rq->calc_load_update + 10)) + this_rq->calc_load_update += LOAD_FREQ; +} + +static long calc_load_fold_idle(void) +{ + int idx = calc_load_read_idx(); + long delta = 0; + + if (atomic_long_read(&calc_load_idle[idx])) + delta = atomic_long_xchg(&calc_load_idle[idx], 0); + + return delta; +} + +/** + * fixed_power_int - compute: x^n, in O(log n) time + * + * @x: base of the power + * @frac_bits: fractional bits of @x + * @n: power to raise @x to. + * + * By exploiting the relation between the definition of the natural power + * function: x^n := x*x*...*x (x multiplied by itself for n times), and + * the binary encoding of numbers used by computers: n := \Sum n_i * 2^i, + * (where: n_i \elem {0, 1}, the binary vector representing n), + * we find: x^n := x^(\Sum n_i * 2^i) := \Prod x^(n_i * 2^i), which is + * of course trivially computable in O(log_2 n), the length of our binary + * vector. + */ +static unsigned long +fixed_power_int(unsigned long x, unsigned int frac_bits, unsigned int n) +{ + unsigned long result = 1UL << frac_bits; + + if (n) for (;;) { + if (n & 1) { + result *= x; + result += 1UL << (frac_bits - 1); + result >>= frac_bits; + } + n >>= 1; + if (!n) + break; + x *= x; + x += 1UL << (frac_bits - 1); + x >>= frac_bits; + } + + return result; +} + +/* + * a1 = a0 * e + a * (1 - e) + * + * a2 = a1 * e + a * (1 - e) + * = (a0 * e + a * (1 - e)) * e + a * (1 - e) + * = a0 * e^2 + a * (1 - e) * (1 + e) + * + * a3 = a2 * e + a * (1 - e) + * = (a0 * e^2 + a * (1 - e) * (1 + e)) * e + a * (1 - e) + * = a0 * e^3 + a * (1 - e) * (1 + e + e^2) + * + * ... + * + * an = a0 * e^n + a * (1 - e) * (1 + e + ... + e^n-1) [1] + * = a0 * e^n + a * (1 - e) * (1 - e^n)/(1 - e) + * = a0 * e^n + a * (1 - e^n) + * + * [1] application of the geometric series: + * + * n 1 - x^(n+1) + * S_n := \Sum x^i = ------------- + * i=0 1 - x + */ +static unsigned long +calc_load_n(unsigned long load, unsigned long exp, + unsigned long active, unsigned int n) +{ + + return calc_load(load, fixed_power_int(exp, FSHIFT, n), active); +} + +/* + * NO_HZ can leave us missing all per-cpu ticks calling + * calc_load_account_active(), but since an idle CPU folds its delta into + * calc_load_tasks_idle per calc_load_account_idle(), all we need to do is fold + * in the pending idle delta if our idle period crossed a load cycle boundary. + * + * Once we've updated the global active value, we need to apply the exponential + * weights adjusted to the number of cycles missed. + */ +static void calc_global_nohz(void) +{ + long delta, active, n; + + if (!time_before(jiffies, calc_load_update + 10)) { + /* + * Catch-up, fold however many we are behind still + */ + delta = jiffies - calc_load_update - 10; + n = 1 + (delta / LOAD_FREQ); + + active = atomic_long_read(&calc_load_tasks); + active = active > 0 ? active * FIXED_1 : 0; + + avenrun[0] = calc_load_n(avenrun[0], EXP_1, active, n); + avenrun[1] = calc_load_n(avenrun[1], EXP_5, active, n); + avenrun[2] = calc_load_n(avenrun[2], EXP_15, active, n); + + calc_load_update += n * LOAD_FREQ; + } + + /* + * Flip the idle index... + * + * Make sure we first write the new time then flip the index, so that + * calc_load_write_idx() will see the new time when it reads the new + * index, this avoids a double flip messing things up. + */ + smp_wmb(); + calc_load_idx++; +} +#else /* !CONFIG_NO_HZ_COMMON */ + +static inline long calc_load_fold_idle(void) { return 0; } +static inline void calc_global_nohz(void) { } + +#endif /* CONFIG_NO_HZ_COMMON */ + +/* + * calc_load - update the avenrun load estimates 10 ticks after the + * CPUs have updated calc_load_tasks. + */ +void calc_global_load(unsigned long ticks) +{ + long active, delta; + + if (time_before(jiffies, calc_load_update + 10)) + return; + + /* + * Fold the 'old' idle-delta to include all NO_HZ cpus. + */ + delta = calc_load_fold_idle(); + if (delta) + atomic_long_add(delta, &calc_load_tasks); + + active = atomic_long_read(&calc_load_tasks); + active = active > 0 ? active * FIXED_1 : 0; + + avenrun[0] = calc_load(avenrun[0], EXP_1, active); + avenrun[1] = calc_load(avenrun[1], EXP_5, active); + avenrun[2] = calc_load(avenrun[2], EXP_15, active); + + calc_load_update += LOAD_FREQ; + + /* + * In case we idled for multiple LOAD_FREQ intervals, catch up in bulk. + */ + calc_global_nohz(); +} + +/* + * Called from update_cpu_load() to periodically update this CPU's + * active count. + */ +static void calc_load_account_active(struct rq *this_rq) +{ + long delta; + + if (time_before(jiffies, this_rq->calc_load_update)) + return; + + delta = calc_load_fold_active(this_rq); + if (delta) + atomic_long_add(delta, &calc_load_tasks); + + this_rq->calc_load_update += LOAD_FREQ; +} + +/* + * End of global load-average stuff + */ + +/* + * The exact cpuload at various idx values, calculated at every tick would be + * load = (2^idx - 1) / 2^idx * load + 1 / 2^idx * cur_load + * + * If a cpu misses updates for n-1 ticks (as it was idle) and update gets called + * on nth tick when cpu may be busy, then we have: + * load = ((2^idx - 1) / 2^idx)^(n-1) * load + * load = (2^idx - 1) / 2^idx) * load + 1 / 2^idx * cur_load + * + * decay_load_missed() below does efficient calculation of + * load = ((2^idx - 1) / 2^idx)^(n-1) * load + * avoiding 0..n-1 loop doing load = ((2^idx - 1) / 2^idx) * load + * + * The calculation is approximated on a 128 point scale. + * degrade_zero_ticks is the number of ticks after which load at any + * particular idx is approximated to be zero. + * degrade_factor is a precomputed table, a row for each load idx. + * Each column corresponds to degradation factor for a power of two ticks, + * based on 128 point scale. + * Example: + * row 2, col 3 (=12) says that the degradation at load idx 2 after + * 8 ticks is 12/128 (which is an approximation of exact factor 3^8/4^8). + * + * With this power of 2 load factors, we can degrade the load n times + * by looking at 1 bits in n and doing as many mult/shift instead of + * n mult/shifts needed by the exact degradation. + */ +#define DEGRADE_SHIFT 7 +static const unsigned char + degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128}; +static const unsigned char + degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = { + {0, 0, 0, 0, 0, 0, 0, 0}, + {64, 32, 8, 0, 0, 0, 0, 0}, + {96, 72, 40, 12, 1, 0, 0}, + {112, 98, 75, 43, 15, 1, 0}, + {120, 112, 98, 76, 45, 16, 2} }; + +/* + * Update cpu_load for any missed ticks, due to tickless idle. The backlog + * would be when CPU is idle and so we just decay the old load without + * adding any new load. + */ +static unsigned long +decay_load_missed(unsigned long load, unsigned long missed_updates, int idx) +{ + int j = 0; + + if (!missed_updates) + return load; + + if (missed_updates >= degrade_zero_ticks[idx]) + return 0; + + if (idx == 1) + return load >> missed_updates; + + while (missed_updates) { + if (missed_updates % 2) + load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT; + + missed_updates >>= 1; + j++; + } + return load; +} + +/* + * Update rq->cpu_load[] statistics. This function is usually called every + * scheduler tick (TICK_NSEC). With tickless idle this will not be called + * every tick. We fix it up based on jiffies. + */ +static void __update_cpu_load(struct rq *this_rq, unsigned long this_load, + unsigned long pending_updates) +{ + int i, scale; + + this_rq->nr_load_updates++; + + /* Update our load: */ + this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */ + for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) { + unsigned long old_load, new_load; + + /* scale is effectively 1 << i now, and >> i divides by scale */ + + old_load = this_rq->cpu_load[i]; + old_load = decay_load_missed(old_load, pending_updates - 1, i); + new_load = this_load; + /* + * Round up the averaging division if load is increasing. This + * prevents us from getting stuck on 9 if the load is 10, for + * example. + */ + if (new_load > old_load) + new_load += scale - 1; + + this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i; + } + + sched_avg_update(this_rq); +} + +#ifdef CONFIG_SMP +static inline unsigned long get_rq_runnable_load(struct rq *rq) +{ + return rq->cfs.runnable_load_avg; +} +#else +static inline unsigned long get_rq_runnable_load(struct rq *rq) +{ + return rq->load.weight; +} +#endif + +#ifdef CONFIG_NO_HZ_COMMON +/* + * There is no sane way to deal with nohz on smp when using jiffies because the + * cpu doing the jiffies update might drift wrt the cpu doing the jiffy reading + * causing off-by-one errors in observed deltas; {0,2} instead of {1,1}. + * + * Therefore we cannot use the delta approach from the regular tick since that + * would seriously skew the load calculation. However we'll make do for those + * updates happening while idle (nohz_idle_balance) or coming out of idle + * (tick_nohz_idle_exit). + * + * This means we might still be one tick off for nohz periods. + */ + +/* + * Called from nohz_idle_balance() to update the load ratings before doing the + * idle balance. + */ +void update_idle_cpu_load(struct rq *this_rq) +{ + unsigned long curr_jiffies = ACCESS_ONCE(jiffies); + unsigned long load = get_rq_runnable_load(this_rq); + unsigned long pending_updates; + + /* + * bail if there's load or we're actually up-to-date. + */ + if (load || curr_jiffies == this_rq->last_load_update_tick) + return; + + pending_updates = curr_jiffies - this_rq->last_load_update_tick; + this_rq->last_load_update_tick = curr_jiffies; + + __update_cpu_load(this_rq, load, pending_updates); +} + +/* + * Called from tick_nohz_idle_exit() -- try and fix up the ticks we missed. + */ +void update_cpu_load_nohz(void) +{ + struct rq *this_rq = this_rq(); + unsigned long curr_jiffies = ACCESS_ONCE(jiffies); + unsigned long pending_updates; + + if (curr_jiffies == this_rq->last_load_update_tick) + return; + + raw_spin_lock(&this_rq->lock); + pending_updates = curr_jiffies - this_rq->last_load_update_tick; + if (pending_updates) { + this_rq->last_load_update_tick = curr_jiffies; + /* + * We were idle, this means load 0, the current load might be + * !0 due to remote wakeups and the sort. + */ + __update_cpu_load(this_rq, 0, pending_updates); + } + raw_spin_unlock(&this_rq->lock); +} +#endif /* CONFIG_NO_HZ */ + +/* + * Called from scheduler_tick() + */ +void update_cpu_load_active(struct rq *this_rq) +{ + unsigned long load = get_rq_runnable_load(this_rq); + /* + * See the mess around update_idle_cpu_load() / update_cpu_load_nohz(). + */ + this_rq->last_load_update_tick = jiffies; + __update_cpu_load(this_rq, load, 1); + + calc_load_account_active(this_rq); +} diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 8b5840c5f1ab1..964211b5c794f 100755 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -6,8 +6,9 @@ #include "sched.h" #include -#include - +#ifdef CONFIG_MTPROF +#include "mt_sched_mon.h" +#endif int sched_rr_timeslice = RR_TIMESLICE; static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun); @@ -28,6 +29,11 @@ static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer) if (!overrun) break; +#ifdef CONFIG_MTPROF + /* mt throttle monitor */ + mt_rt_mon_switch(MON_RESET); + mt_rt_mon_switch(MON_START); +#endif idle = do_sched_rt_period_timer(rt_b, overrun); } @@ -80,6 +86,8 @@ void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq) rt_rq->overloaded = 0; plist_head_init(&rt_rq->pushable_tasks); #endif + /* We start is dequeued state, because no RT tasks are queued */ + rt_rq->rt_queued = 0; rt_rq->rt_time = 0; rt_rq->rt_throttled = 0; @@ -113,6 +121,13 @@ static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) return rt_se->rt_rq; } +static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) +{ + struct rt_rq *rt_rq = rt_se->rt_rq; + + return rt_rq->rq; +} + void free_rt_sched_group(struct task_group *tg) { int i; @@ -212,10 +227,16 @@ static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) return container_of(rt_rq, struct rq, rt); } -static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) +static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) { struct task_struct *p = rt_task_of(rt_se); - struct rq *rq = task_rq(p); + + return task_rq(p); +} + +static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) +{ + struct rq *rq = rq_of_rt_se(rt_se); return &rq->rt; } @@ -230,6 +251,14 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) #ifdef CONFIG_SMP +static int pull_rt_task(struct rq *this_rq); + +static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev) +{ + /* Try to pull RT tasks here if we lower this rq's prio */ + return rq->rt.highest_prio.curr > prev->prio; +} + static inline int rt_overloaded(struct rq *rq) { return atomic_read(&rq->rd->rto_count); @@ -247,8 +276,10 @@ static inline void rt_set_overload(struct rq *rq) * if we should look at the mask. It would be a shame * if we looked at the mask, but the mask was not * updated yet. + * + * Matched by the barrier in pull_rt_task(). */ - wmb(); + smp_wmb(); atomic_inc(&rq->rd->rto_count); } @@ -314,6 +345,15 @@ static inline int has_pushable_tasks(struct rq *rq) return !plist_head_empty(&rq->rt.pushable_tasks); } +static inline void set_post_schedule(struct rq *rq) +{ + /* + * We detect this state here so that we can avoid taking the RQ + * lock again later if there is no need to push + */ + rq->post_schedule = has_pushable_tasks(rq); +} + static void enqueue_pushable_task(struct rq *rq, struct task_struct *p) { plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); @@ -358,8 +398,24 @@ void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) { } +static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev) +{ + return false; +} + +static inline int pull_rt_task(struct rq *this_rq) +{ + return 0; +} + +static inline void set_post_schedule(struct rq *rq) +{ +} #endif /* CONFIG_SMP */ +static void enqueue_top_rt_rq(struct rt_rq *rt_rq); +static void dequeue_top_rt_rq(struct rt_rq *rt_rq); + static inline int on_rt_rq(struct sched_rt_entity *rt_se) { return !list_empty(&rt_se->run_list); @@ -400,20 +456,6 @@ static inline struct task_group *next_task_group(struct task_group *tg) (iter = next_task_group(iter)) && \ (rt_rq = iter->rt_rq[cpu_of(rq)]);) -static inline void list_add_leaf_rt_rq(struct rt_rq *rt_rq) -{ - list_add_rcu(&rt_rq->leaf_rt_rq_list, - &rq_of_rt_rq(rt_rq)->leaf_rt_rq_list); -} - -static inline void list_del_leaf_rt_rq(struct rt_rq *rt_rq) -{ - list_del_rcu(&rt_rq->leaf_rt_rq_list); -} - -#define for_each_leaf_rt_rq(rt_rq, rq) \ - list_for_each_entry_rcu(rt_rq, &rq->leaf_rt_rq_list, leaf_rt_rq_list) - #define for_each_sched_rt_entity(rt_se) \ for (; rt_se; rt_se = rt_se->parent) @@ -428,17 +470,21 @@ static void dequeue_rt_entity(struct sched_rt_entity *rt_se); static void sched_rt_rq_enqueue(struct rt_rq *rt_rq) { struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr; + struct rq *rq = rq_of_rt_rq(rt_rq); struct sched_rt_entity *rt_se; - int cpu = cpu_of(rq_of_rt_rq(rt_rq)); + int cpu = cpu_of(rq); rt_se = rt_rq->tg->rt_se[cpu]; if (rt_rq->rt_nr_running) { - if (rt_se && !on_rt_rq(rt_se)) + if (!rt_se) + enqueue_top_rt_rq(rt_rq); + else if (!on_rt_rq(rt_se)) enqueue_rt_entity(rt_se, false); + if (rt_rq->highest_prio.curr < curr->prio) - resched_task(curr); + resched_curr(rq); } } @@ -449,7 +495,9 @@ static void sched_rt_rq_dequeue(struct rt_rq *rt_rq) rt_se = rt_rq->tg->rt_se[cpu]; - if (rt_se && on_rt_rq(rt_se)) + if (!rt_se) + dequeue_top_rt_rq(rt_rq); + else if (on_rt_rq(rt_se)) dequeue_rt_entity(rt_se); } @@ -473,7 +521,7 @@ static int rt_se_boosted(struct sched_rt_entity *rt_se) #ifdef CONFIG_SMP static inline const struct cpumask *sched_rt_period_mask(void) { - return cpu_rq(smp_processor_id())->rd->span; + return this_rq()->rd->span; } #else static inline const struct cpumask *sched_rt_period_mask(void) @@ -510,17 +558,6 @@ typedef struct rt_rq *rt_rq_iter_t; #define for_each_rt_rq(rt_rq, iter, rq) \ for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL) -static inline void list_add_leaf_rt_rq(struct rt_rq *rt_rq) -{ -} - -static inline void list_del_leaf_rt_rq(struct rt_rq *rt_rq) -{ -} - -#define for_each_leaf_rt_rq(rt_rq, rq) \ - for (rt_rq = &rq->rt; rt_rq; rt_rq = NULL) - #define for_each_sched_rt_entity(rt_se) \ for (; rt_se; rt_se = NULL) @@ -531,12 +568,18 @@ static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq) { - if (rt_rq->rt_nr_running) - resched_task(rq_of_rt_rq(rt_rq)->curr); + struct rq *rq = rq_of_rt_rq(rt_rq); + + if (!rt_rq->rt_nr_running) + return; + + enqueue_top_rt_rq(rt_rq); + resched_curr(rq); } static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq) { + dequeue_top_rt_rq(rt_rq); } static inline int rt_rq_throttled(struct rt_rq *rt_rq) @@ -562,6 +605,14 @@ static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq) #endif /* CONFIG_RT_GROUP_SCHED */ +bool sched_rt_bandwidth_account(struct rt_rq *rt_rq) +{ + struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); + + return (hrtimer_active(&rt_b->rt_period_timer) || + rt_rq->rt_time < rt_b->rt_runtime); +} + #ifdef CONFIG_SMP /* * We ran out of runtime, see if we can borrow some from our neighbours. @@ -697,6 +748,9 @@ static void __disable_runtime(struct rq *rq) rt_rq->rt_throttled = 0; raw_spin_unlock(&rt_rq->rt_runtime_lock); raw_spin_unlock(&rt_b->rt_runtime_lock); + + /* Make rt_rq available for pick_next_task() */ + sched_rt_rq_enqueue(rt_rq); } } @@ -827,51 +881,6 @@ static inline int rt_se_prio(struct sched_rt_entity *rt_se) return rt_task_of(rt_se)->prio; } -static void dump_throttled_rt_tasks(struct rt_rq *rt_rq) -{ - struct rt_prio_array *array = &rt_rq->active; - struct sched_rt_entity *rt_se; - char buf[500]; - char *pos = buf; - char *end = buf + sizeof(buf); - int idx; - - pos += snprintf(pos, sizeof(buf), - "sched: RT throttling activated for rt_rq %p (cpu %d)\n", - rt_rq, cpu_of(rq_of_rt_rq(rt_rq))); - - if (bitmap_empty(array->bitmap, MAX_RT_PRIO)) - goto out; - - pos += snprintf(pos, end - pos, "potential CPU hogs:\n"); - idx = sched_find_first_bit(array->bitmap); - while (idx < MAX_RT_PRIO) { - list_for_each_entry(rt_se, array->queue + idx, run_list) { - struct task_struct *p; - - if (!rt_entity_is_task(rt_se)) - continue; - - p = rt_task_of(rt_se); - if (pos < end) - pos += snprintf(pos, end - pos, "\t%s (%d)\n", - p->comm, p->pid); - } - idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx + 1); - } -out: -#ifdef CONFIG_PANIC_ON_RT_THROTTLING - /* - * Use pr_err() in the BUG() case since printk_sched() will - * not get flushed and deadlock is not a concern. - */ - pr_err("%s", buf); - BUG(); -#else - printk_deferred("%s", buf); -#endif -} - static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) { u64 runtime = sched_rt_runtime(rt_rq); @@ -895,14 +904,15 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) * but accrue some time due to boosting. */ if (likely(rt_b->rt_runtime)) { - static bool once = false; - rt_rq->rt_throttled = 1; - - if (!once) { - once = true; - dump_throttled_rt_tasks(rt_rq); - } +#if 0 + printk_deferred_once("sched: RT throttling activated\n"); +#endif +#ifdef CONFIG_MTPROF + /* sched:rt throttle monitor */ + mt_rt_mon_switch(MON_STOP); + mt_rt_mon_print_task(); +#endif } else { /* * In case we did anyway, make it go away, @@ -929,13 +939,12 @@ static void update_curr_rt(struct rq *rq) { struct task_struct *curr = rq->curr; struct sched_rt_entity *rt_se = &curr->rt; - struct rt_rq *rt_rq = rt_rq_of_se(rt_se); u64 delta_exec; if (curr->sched_class != &rt_sched_class) return; - delta_exec = rq->clock_task - curr->se.exec_start; + delta_exec = rq_clock_task(rq) - curr->se.exec_start; if (unlikely((s64)delta_exec <= 0)) return; @@ -945,7 +954,7 @@ static void update_curr_rt(struct rq *rq) curr->se.sum_exec_runtime += delta_exec; account_group_exec_runtime(curr, delta_exec); - curr->se.exec_start = rq->clock_task; + curr->se.exec_start = rq_clock_task(rq); cpuacct_charge(curr, delta_exec); sched_rt_avg_update(rq, delta_exec); @@ -954,18 +963,50 @@ static void update_curr_rt(struct rq *rq) return; for_each_sched_rt_entity(rt_se) { - rt_rq = rt_rq_of_se(rt_se); + struct rt_rq *rt_rq = rt_rq_of_se(rt_se); if (sched_rt_runtime(rt_rq) != RUNTIME_INF) { raw_spin_lock(&rt_rq->rt_runtime_lock); rt_rq->rt_time += delta_exec; if (sched_rt_runtime_exceeded(rt_rq)) - resched_task(curr); + resched_curr(rq); raw_spin_unlock(&rt_rq->rt_runtime_lock); } } } +static void +dequeue_top_rt_rq(struct rt_rq *rt_rq) +{ + struct rq *rq = rq_of_rt_rq(rt_rq); + + BUG_ON(&rq->rt != rt_rq); + + if (!rt_rq->rt_queued) + return; + + BUG_ON(!rq->nr_running); + + sub_nr_running(rq, rt_rq->rt_nr_running); + rt_rq->rt_queued = 0; +} + +static void +enqueue_top_rt_rq(struct rt_rq *rt_rq) +{ + struct rq *rq = rq_of_rt_rq(rt_rq); + + BUG_ON(&rq->rt != rt_rq); + + if (rt_rq->rt_queued) + return; + if (rt_rq_throttled(rt_rq) || !rt_rq->rt_nr_running) + return; + + add_nr_running(rq, rt_rq->rt_nr_running); + rt_rq->rt_queued = 1; +} + #if defined CONFIG_SMP static void @@ -1088,37 +1129,24 @@ void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {} #endif /* CONFIG_RT_GROUP_SCHED */ -#ifdef CONFIG_SCHED_HMP - -static void -inc_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p) +static inline +unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se) { - inc_cumulative_runnable_avg(&rq->hmp_stats, p); -} + struct rt_rq *group_rq = group_rt_rq(rt_se); -static void -dec_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p) -{ - dec_cumulative_runnable_avg(&rq->hmp_stats, p); + if (group_rq) + return group_rq->rt_nr_running; + else + return 1; } -#else /* CONFIG_SCHED_HMP */ - -static inline void -inc_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p) { } - -static inline void -dec_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p) { } - -#endif /* CONFIG_SCHED_HMP */ - static inline void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) { int prio = rt_se_prio(rt_se); WARN_ON(!rt_prio(prio)); - rt_rq->rt_nr_running++; + rt_rq->rt_nr_running += rt_se_nr_running(rt_se); inc_rt_prio(rt_rq, prio); inc_rt_migration(rt_se, rt_rq); @@ -1130,7 +1158,7 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) { WARN_ON(!rt_prio(rt_se_prio(rt_se))); WARN_ON(!rt_rq->rt_nr_running); - rt_rq->rt_nr_running--; + rt_rq->rt_nr_running -= rt_se_nr_running(rt_se); dec_rt_prio(rt_rq, rt_se_prio(rt_se)); dec_rt_migration(rt_se, rt_rq); @@ -1153,9 +1181,6 @@ static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head) if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running)) return; - if (!rt_rq->rt_nr_running) - list_add_leaf_rt_rq(rt_rq); - if (head) list_add(&rt_se->run_list, queue); else @@ -1175,8 +1200,6 @@ static void __dequeue_rt_entity(struct sched_rt_entity *rt_se) __clear_bit(rt_se_prio(rt_se), array->bitmap); dec_rt_tasks(rt_se, rt_rq); - if (!rt_rq->rt_nr_running) - list_del_leaf_rt_rq(rt_rq); } /* @@ -1192,6 +1215,8 @@ static void dequeue_rt_stack(struct sched_rt_entity *rt_se) back = rt_se; } + dequeue_top_rt_rq(rt_rq_of_se(back)); + for (rt_se = back; rt_se; rt_se = rt_se->back) { if (on_rt_rq(rt_se)) __dequeue_rt_entity(rt_se); @@ -1200,13 +1225,18 @@ static void dequeue_rt_stack(struct sched_rt_entity *rt_se) static void enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head) { + struct rq *rq = rq_of_rt_se(rt_se); + dequeue_rt_stack(rt_se); for_each_sched_rt_entity(rt_se) __enqueue_rt_entity(rt_se, head); + enqueue_top_rt_rq(&rq->rt); } static void dequeue_rt_entity(struct sched_rt_entity *rt_se) { + struct rq *rq = rq_of_rt_se(rt_se); + dequeue_rt_stack(rt_se); for_each_sched_rt_entity(rt_se) { @@ -1215,6 +1245,7 @@ static void dequeue_rt_entity(struct sched_rt_entity *rt_se) if (rt_rq && rt_rq->rt_nr_running) __enqueue_rt_entity(rt_se, false); } + enqueue_top_rt_rq(&rq->rt); } /* @@ -1232,9 +1263,6 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags) if (!task_current(rq, p) && p->nr_cpus_allowed > 1) enqueue_pushable_task(rq, p); - - inc_nr_running(rq); - inc_hmp_sched_stats_rt(rq, p); } static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags) @@ -1245,9 +1273,6 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags) dequeue_rt_entity(rt_se); dequeue_pushable_task(rq, p); - - dec_nr_running(rq); - dec_hmp_sched_stats_rt(rq, p); } /* @@ -1288,36 +1313,14 @@ static void yield_task_rt(struct rq *rq) static int find_lowest_rq(struct task_struct *task); static int -select_task_rq_rt_hmp(struct task_struct *p, int sd_flag, int flags) -{ - int cpu, target; - - cpu = task_cpu(p); - - rcu_read_lock(); - target = find_lowest_rq(p); - if (target != -1) - cpu = target; - rcu_read_unlock(); - - return cpu; -} - -static int -select_task_rq_rt(struct task_struct *p, int sd_flag, int flags) +select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags) { struct task_struct *curr; struct rq *rq; - int cpu; - - cpu = task_cpu(p); if (p->nr_cpus_allowed == 1) goto out; - if (sched_enable_hmp) - return select_task_rq_rt_hmp(p, sd_flag, flags); - /* For anything but wake ups, just return the task_cpu */ if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK) goto out; @@ -1351,8 +1354,7 @@ select_task_rq_rt(struct task_struct *p, int sd_flag, int flags) */ if (curr && unlikely(rt_task(curr)) && (curr->nr_cpus_allowed < 2 || - curr->prio <= p->prio) && - (p->nr_cpus_allowed > 1)) { + curr->prio <= p->prio)) { int target = find_lowest_rq(p); if (target != -1) @@ -1382,7 +1384,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p) * to try and push current away: */ requeue_task_rt(rq, p, 1); - resched_task(rq->curr); + resched_curr(rq); } #endif /* CONFIG_SMP */ @@ -1393,7 +1395,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p) static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flags) { if (p->prio < rq->curr->prio) { - resched_task(rq->curr); + resched_curr(rq); return; } @@ -1436,15 +1438,7 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq) { struct sched_rt_entity *rt_se; struct task_struct *p; - struct rt_rq *rt_rq; - - rt_rq = &rq->rt; - - if (!rt_rq->rt_nr_running) - return NULL; - - if (rt_rq_throttled(rt_rq)) - return NULL; + struct rt_rq *rt_rq = &rq->rt; do { rt_se = pick_next_rt_entity(rq, rt_rq); @@ -1452,36 +1446,48 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq) rt_rq = group_rt_rq(rt_se); } while (rt_rq); - /* - * Force update of rq->clock_task in case we failed to do so in - * put_prev_task. A stale value can cause us to over-charge execution - * time to real-time task, that could trigger throttling unnecessarily - */ - if (rq->skip_clock_update > 0) { - rq->skip_clock_update = 0; - update_rq_clock(rq); - } p = rt_task_of(rt_se); - p->se.exec_start = rq->clock_task; + p->se.exec_start = rq_clock_task(rq); return p; } -static struct task_struct *pick_next_task_rt(struct rq *rq) +static struct task_struct * +pick_next_task_rt(struct rq *rq, struct task_struct *prev) { - struct task_struct *p = _pick_next_task_rt(rq); + struct task_struct *p; + struct rt_rq *rt_rq = &rq->rt; - /* The running task is never eligible for pushing */ - if (p) - dequeue_pushable_task(rq, p); + if (need_pull_rt_task(rq, prev)) { + pull_rt_task(rq); + /* + * pull_rt_task() can drop (and re-acquire) rq->lock; this + * means a dl or stop task can slip in, in which case we need + * to re-start task selection. + */ + if (unlikely((rq->stop && task_on_rq_queued(rq->stop)) || + rq->dl.dl_nr_running)) + return RETRY_TASK; + } -#ifdef CONFIG_SMP /* - * We detect this state here so that we can avoid taking the RQ - * lock again later if there is no need to push + * We may dequeue prev's rt_rq in put_prev_task(). + * So, we update time before rt_nr_running check. */ - rq->post_schedule = has_pushable_tasks(rq); -#endif + if (prev->sched_class == &rt_sched_class) + update_curr_rt(rq); + + if (!rt_rq->rt_queued) + return NULL; + + put_prev_task(rq, prev); + + p = _pick_next_task_rt(rq); + + /* The running task is never eligible for pushing */ + dequeue_pushable_task(rq, p); + + set_post_schedule(rq); return p; } @@ -1511,111 +1517,35 @@ static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu) return 0; } -/* Return the second highest RT task, NULL otherwise */ -static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu) +/* + * Return the highest pushable rq's task, which is suitable to be executed + * on the cpu, NULL otherwise + */ +static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu) { - struct task_struct *next = NULL; - struct sched_rt_entity *rt_se; - struct rt_prio_array *array; - struct rt_rq *rt_rq; - int idx; - - for_each_leaf_rt_rq(rt_rq, rq) { - array = &rt_rq->active; - idx = sched_find_first_bit(array->bitmap); -next_idx: - if (idx >= MAX_RT_PRIO) - continue; - if (next && next->prio <= idx) - continue; - list_for_each_entry(rt_se, array->queue + idx, run_list) { - struct task_struct *p; + struct plist_head *head = &rq->rt.pushable_tasks; + struct task_struct *p; - if (!rt_entity_is_task(rt_se)) - continue; + if (!has_pushable_tasks(rq)) + return NULL; - p = rt_task_of(rt_se); - if (pick_rt_task(rq, p, cpu)) { - next = p; - break; - } - } - if (!next) { - idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx+1); - goto next_idx; - } + plist_for_each_entry(p, head, pushable_tasks) { + if (pick_rt_task(rq, p, cpu)) + return p; } - return next; + return NULL; } static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask); -#ifdef CONFIG_SCHED_HMP -static int find_lowest_rq_hmp(struct task_struct *task) -{ - struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask); - int cpu_cost, min_cost = INT_MAX; - int best_cpu = -1; - int i; - - /* Make sure the mask is initialized first */ - if (unlikely(!lowest_mask)) - return best_cpu; - - if (task->nr_cpus_allowed == 1) - return best_cpu; /* No other targets possible */ - - if (!cpupri_find(&task_rq(task)->rd->cpupri, task, lowest_mask)) - return best_cpu; /* No targets found */ - - /* - * At this point we have built a mask of cpus representing the - * lowest priority tasks in the system. Now we want to elect - * the best one based on our affinity and topology. - */ - - /* Skip performance considerations and optimize for power. - * Worst case we'll be iterating over all CPUs here. CPU - * online mask should be taken care of when constructing - * the lowest_mask. - */ - for_each_cpu(i, lowest_mask) { - struct rq *rq = cpu_rq(i); - cpu_cost = power_cost_at_freq(i, ACCESS_ONCE(rq->min_freq)); - trace_sched_cpu_load(rq, idle_cpu(i), mostly_idle_cpu(i), - sched_irqload(i), cpu_cost, cpu_temp(i)); - - if (sched_boost() && capacity(rq) != max_capacity) - continue; - - if (cpu_cost < min_cost && !sched_cpu_high_irqload(i)) { - min_cost = cpu_cost; - best_cpu = i; - } - } - return best_cpu; -} - -#else /* CONFIG_SCHED_HMP */ - -static int find_lowest_rq_hmp(struct task_struct *task) -{ - return -1; -} - -#endif /* CONFIG_SCHED_HMP */ - static int find_lowest_rq(struct task_struct *task) { struct sched_domain *sd; - struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask); + struct cpumask *lowest_mask = this_cpu_ptr(local_cpu_mask); int this_cpu = smp_processor_id(); int cpu = task_cpu(task); - if (sched_enable_hmp) - return find_lowest_rq_hmp(task); - /* Make sure the mask is initialized first */ if (unlikely(!lowest_mask)) return -1; @@ -1710,7 +1640,7 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq) !cpumask_test_cpu(lowest_rq->cpu, tsk_cpus_allowed(task)) || task_running(rq, task) || - !task->on_rq)) { + !task_on_rq_queued(task))) { double_unlock_balance(rq, lowest_rq); lowest_rq = NULL; @@ -1744,7 +1674,7 @@ static struct task_struct *pick_next_pushable_task(struct rq *rq) BUG_ON(task_current(rq, p)); BUG_ON(p->nr_cpus_allowed <= 1); - BUG_ON(!p->on_rq); + BUG_ON(!task_on_rq_queued(p)); BUG_ON(!rt_task(p)); return p; @@ -1780,7 +1710,7 @@ static int push_rt_task(struct rq *rq) * just reschedule current. */ if (unlikely(next_task->prio < rq->curr->prio)) { - resched_task(rq->curr); + resched_curr(rq); return 0; } @@ -1827,7 +1757,7 @@ static int push_rt_task(struct rq *rq) activate_task(lowest_rq, next_task, 0); ret = 1; - resched_task(lowest_rq->curr); + resched_curr(lowest_rq); double_unlock_balance(rq, lowest_rq); @@ -1853,6 +1783,12 @@ static int pull_rt_task(struct rq *this_rq) if (likely(!rt_overloaded(this_rq))) return 0; + /* + * Match the barrier from rt_set_overloaded; this guarantees that if we + * see overloaded we must also see the rto_mask bit. + */ + smp_rmb(); + for_each_cpu(cpu, this_rq->rd->rto_mask) { if (this_cpu == cpu) continue; @@ -1878,12 +1814,10 @@ static int pull_rt_task(struct rq *this_rq) double_lock_balance(this_rq, src_rq); /* - * Are there still pullable RT tasks? + * We can pull only a task, which is pushable + * on its rq, and no others. */ - if (src_rq->rt.rt_nr_running <= 1) - goto skip; - - p = pick_next_highest_task_rt(src_rq, this_cpu); + p = pick_highest_pushable_task(src_rq, this_cpu); /* * Do we have an RT task that preempts @@ -1891,7 +1825,7 @@ static int pull_rt_task(struct rq *this_rq) */ if (p && (p->prio < this_rq->rt.highest_prio.curr)) { WARN_ON(p == src_rq->curr); - WARN_ON(!p->on_rq); + WARN_ON(!task_on_rq_queued(p)); /* * There's a chance that p is higher in priority @@ -1923,13 +1857,6 @@ static int pull_rt_task(struct rq *this_rq) return ret; } -static void pre_schedule_rt(struct rq *rq, struct task_struct *prev) -{ - /* Try to pull RT tasks here if we lower this rq's prio */ - if (rq->rt.highest_prio.curr > prev->prio) - pull_rt_task(rq); -} - static void post_schedule_rt(struct rq *rq) { push_rt_tasks(rq); @@ -1945,7 +1872,7 @@ static void task_woken_rt(struct rq *rq, struct task_struct *p) !test_tsk_need_resched(rq->curr) && has_pushable_tasks(rq) && p->nr_cpus_allowed > 1 && - rt_task(rq->curr) && + (dl_task(rq->curr) || rt_task(rq->curr)) && (rq->curr->nr_cpus_allowed < 2 || rq->curr->prio <= p->prio)) push_rt_tasks(rq); @@ -1959,7 +1886,7 @@ static void set_cpus_allowed_rt(struct task_struct *p, BUG_ON(!rt_task(p)); - if (!p->on_rq) + if (!task_on_rq_queued(p)) return; weight = cpumask_weight(new_mask); @@ -2025,14 +1952,14 @@ static void switched_from_rt(struct rq *rq, struct task_struct *p) * we may need to handle the pulling of RT tasks * now. */ - if (!p->on_rq || rq->rt.rt_nr_running) + if (!task_on_rq_queued(p) || rq->rt.rt_nr_running) return; if (pull_rt_task(rq)) - resched_task(rq->curr); + resched_curr(rq); } -void init_sched_rt_class(void) +void __init init_sched_rt_class(void) { unsigned int i; @@ -2041,7 +1968,6 @@ void init_sched_rt_class(void) GFP_KERNEL, cpu_to_node(i)); } } - #endif /* CONFIG_SMP */ /* @@ -2060,15 +1986,15 @@ static void switched_to_rt(struct rq *rq, struct task_struct *p) * If that current running task is also an RT task * then see if we can move to another run queue. */ - if (p->on_rq && rq->curr != p) { + if (task_on_rq_queued(p) && rq->curr != p) { #ifdef CONFIG_SMP - if (rq->rt.overloaded && push_rt_task(rq) && + if (p->nr_cpus_allowed > 1 && rq->rt.overloaded && /* Don't resched if we changed runqueues */ - rq != task_rq(p)) + push_rt_task(rq) && rq != task_rq(p)) check_resched = 0; #endif /* CONFIG_SMP */ if (check_resched && p->prio < rq->curr->prio) - resched_task(rq->curr); + resched_curr(rq); } } @@ -2079,7 +2005,7 @@ static void switched_to_rt(struct rq *rq, struct task_struct *p) static void prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio) { - if (!p->on_rq) + if (!task_on_rq_queued(p)) return; if (rq->curr == p) { @@ -2097,11 +2023,11 @@ prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio) * Only reschedule if p is still on the same runqueue. */ if (p->prio > rq->rt.highest_prio.curr && rq->curr == p) - resched_task(p); + resched_curr(rq); #else /* For UP simply resched on drop of prio */ if (oldprio < p->prio) - resched_task(p); + resched_curr(rq); #endif /* CONFIG_SMP */ } else { /* @@ -2110,7 +2036,7 @@ prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio) * then reschedule. */ if (p->prio < rq->curr->prio) - resched_task(rq->curr); + resched_curr(rq); } } @@ -2157,13 +2083,13 @@ static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued) p->rt.time_slice = sched_rr_timeslice; /* - * Requeue to the end of queue if we (and all of our ancestors) are the - * only element on the queue + * Requeue to the end of queue if we (and all of our ancestors) are not + * the only element on the queue */ for_each_sched_rt_entity(rt_se) { if (rt_se->run_list.prev != rt_se->run_list.next) { requeue_task_rt(rq, p, 0); - set_tsk_need_resched(p); + resched_curr(rq); return; } } @@ -2173,7 +2099,7 @@ static void set_curr_task_rt(struct rq *rq) { struct task_struct *p = rq->curr; - p->se.exec_start = rq->clock_task; + p->se.exec_start = rq_clock_task(rq); /* The running task is never eligible for pushing */ dequeue_pushable_task(rq, p); @@ -2207,7 +2133,6 @@ const struct sched_class rt_sched_class = { .set_cpus_allowed = set_cpus_allowed_rt, .rq_online = rq_online_rt, .rq_offline = rq_offline_rt, - .pre_schedule = pre_schedule_rt, .post_schedule = post_schedule_rt, .task_woken = task_woken_rt, .switched_from = switched_from_rt, @@ -2220,10 +2145,8 @@ const struct sched_class rt_sched_class = { .prio_changed = prio_changed_rt, .switched_to = switched_to_rt, -#ifdef CONFIG_SCHED_HMP - .inc_hmp_sched_stats = inc_hmp_sched_stats_rt, - .dec_hmp_sched_stats = dec_hmp_sched_stats_rt, -#endif + + .update_curr = update_curr_rt, }; #ifdef CONFIG_SCHED_DEBUG diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index a47b87294572d..ed2201b362e70 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2,33 +2,31 @@ #include #include #include +#include #include #include #include #include +#include #include "cpupri.h" +#include "cpudeadline.h" #include "cpuacct.h" +struct rq; +struct cpuidle_state; + +/* task_struct::on_rq states: */ +#define TASK_ON_RQ_QUEUED 1 +#define TASK_ON_RQ_MIGRATING 2 + extern __read_mostly int scheduler_running; -/* - * Convert user-nice values [ -20 ... 0 ... 19 ] - * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ], - * and back. - */ -#define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20) -#define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20) -#define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio) +extern unsigned long calc_load_update; +extern atomic_long_t calc_load_tasks; -/* - * 'User priority' is the nice value converted to something we - * can work with better when scaling various scheduler parameters, - * it's a [ 0 ... 39 ] range. - */ -#define USER_PRIO(p) ((p)-MAX_RT_PRIO) -#define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio) -#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO)) +extern long calc_load_fold_active(struct rq *this_rq); +extern void update_cpu_load_active(struct rq *this_rq); /* * Helpers for converting nanosecond timing to jiffy resolution @@ -63,6 +61,13 @@ extern __read_mostly int scheduler_running; #define NICE_0_LOAD SCHED_LOAD_SCALE #define NICE_0_SHIFT SCHED_LOAD_SHIFT +/* + * Single value that decides SCHED_DEADLINE internal math precision. + * 10 -> just above 1us + * 9 -> just above 0.5us + */ +#define DL_SCALE (10) + /* * These are the 'tuning knobs' of the scheduler: */ @@ -82,11 +87,35 @@ static inline int rt_policy(int policy) return policy == SCHED_FIFO || policy == SCHED_RR; } +static inline int dl_policy(int policy) +{ + return policy == SCHED_DEADLINE; +} + static inline int task_has_rt_policy(struct task_struct *p) { return rt_policy(p->policy); } +static inline int task_has_dl_policy(struct task_struct *p) +{ + return dl_policy(p->policy); +} + +static inline bool dl_time_before(u64 a, u64 b) +{ + return (s64)(a - b) < 0; +} + +/* + * Tells if entity @a should preempt entity @b. + */ +static inline bool +dl_entity_preempt(struct sched_dl_entity *a, struct sched_dl_entity *b) +{ + return dl_time_before(a->deadline, b->deadline); +} + /* * This is the priority-queue data structure of the RT scheduling class: */ @@ -103,6 +132,50 @@ struct rt_bandwidth { struct hrtimer rt_period_timer; }; +void __dl_clear_params(struct task_struct *p); + +/* + * To keep the bandwidth of -deadline tasks and groups under control + * we need some place where: + * - store the maximum -deadline bandwidth of the system (the group); + * - cache the fraction of that bandwidth that is currently allocated. + * + * This is all done in the data structure below. It is similar to the + * one used for RT-throttling (rt_bandwidth), with the main difference + * that, since here we are only interested in admission control, we + * do not decrease any runtime while the group "executes", neither we + * need a timer to replenish it. + * + * With respect to SMP, the bandwidth is given on a per-CPU basis, + * meaning that: + * - dl_bw (< 100%) is the bandwidth of the system (group) on each CPU; + * - dl_total_bw array contains, in the i-eth element, the currently + * allocated bandwidth on the i-eth CPU. + * Moreover, groups consume bandwidth on each CPU, while tasks only + * consume bandwidth on the CPU they're running on. + * Finally, dl_total_bw_cpu is used to cache the index of dl_total_bw + * that will be shown the next time the proc or cgroup controls will + * be red. It on its turn can be changed by writing on its own + * control. + */ +struct dl_bandwidth { + raw_spinlock_t dl_runtime_lock; + u64 dl_runtime; + u64 dl_period; +}; + +static inline int dl_bandwidth_enabled(void) +{ + return sysctl_sched_rt_runtime >= 0; +} + +extern struct dl_bw *dl_bw_of(int i); + +struct dl_bw { + raw_spinlock_t lock; + u64 bw, total_bw; +}; + extern struct mutex sched_domains_mutex; #ifdef CONFIG_CGROUP_SCHED @@ -125,7 +198,7 @@ struct cfs_bandwidth { #ifdef VENDOR_EDIT u64 quota_per_task; #endif - s64 hierarchal_quota; + s64 hierarchical_quota; u64 runtime_expires; int idle, timer_active; @@ -146,11 +219,6 @@ struct cfs_bandwidth { struct task_group { struct cgroup_subsys_state css; - bool notify_on_migrate; -#ifdef CONFIG_SCHED_HMP - bool upmigrate_discouraged; -#endif - #ifdef CONFIG_FAIR_GROUP_SCHED /* schedulable entities of this group on each cpu */ struct sched_entity **se; @@ -158,10 +226,11 @@ struct task_group { struct cfs_rq **cfs_rq; unsigned long shares; - atomic_t load_weight; - atomic64_t load_avg; +#ifdef CONFIG_SMP + atomic_long_t load_avg; atomic_t runnable_avg; #endif +#endif #ifdef CONFIG_RT_GROUP_SCHED struct sched_rt_entity **rt_se; @@ -254,15 +323,6 @@ struct cfs_bandwidth { }; #endif /* CONFIG_CGROUP_SCHED */ -#ifdef CONFIG_SCHED_HMP - -struct hmp_sched_stats { - int nr_big_tasks, nr_small_tasks; - u64 cumulative_runnable_avg; -}; - -#endif - /* CFS-related fields in a runqueue */ struct cfs_rq { struct load_weight load; @@ -288,27 +348,31 @@ struct cfs_rq { #endif #ifdef CONFIG_SMP -/* - * Load-tracking only depends on SMP, FAIR_GROUP_SCHED dependency below may be - * removed when useful for applications beyond shares distribution (e.g. - * load-balance). - */ -#ifdef CONFIG_FAIR_GROUP_SCHED /* * CFS Load tracking * Under CFS, load is tracked on a per-entity basis and aggregated up. * This allows for the description of both thread and group usage (in * the FAIR_GROUP_SCHED case). + * runnable_load_avg is the sum of the load_avg_contrib of the + * sched_entities on the rq. + * blocked_load_avg is similar to runnable_load_avg except that its + * the blocked sched_entities on the rq. + * utilization_load_avg is the sum of the average running time of the + * sched_entities on the rq. + * utilization_blocked_avg is the utilization equivalent of + * blocked_load_avg, i.e. the sum of running contributions of blocked + * sched_entities associated with the rq. */ - u64 runnable_load_avg, blocked_load_avg; - atomic64_t decay_counter, removed_load; + unsigned long runnable_load_avg, blocked_load_avg; + unsigned long utilization_load_avg, utilization_blocked_avg; + atomic64_t decay_counter; u64 last_decay; -#endif /* CONFIG_FAIR_GROUP_SCHED */ -/* These always depend on CONFIG_FAIR_GROUP_SCHED */ + atomic_long_t removed_load, removed_utilization; + #ifdef CONFIG_FAIR_GROUP_SCHED + /* Required to track per-cpu representation of a task_group */ u32 tg_runnable_contrib; - u64 tg_load_contrib; -#endif /* CONFIG_FAIR_GROUP_SCHED */ + unsigned long tg_load_contrib; /* * h_load = weight * f(tg) @@ -317,6 +381,9 @@ struct cfs_rq { * this group. */ unsigned long h_load; + u64 last_h_load_update; + struct sched_entity *h_load_next; +#endif /* CONFIG_FAIR_GROUP_SCHED */ #endif /* CONFIG_SMP */ #ifdef CONFIG_FAIR_GROUP_SCHED @@ -335,11 +402,6 @@ struct cfs_rq { struct task_group *tg; /* group that "owns" this runqueue */ #ifdef CONFIG_CFS_BANDWIDTH - -#ifdef CONFIG_SCHED_HMP - struct hmp_sched_stats hmp_stats; -#endif - int runtime_enabled; u64 runtime_expires; s64 runtime_remaining; @@ -380,6 +442,8 @@ struct rt_rq { int overloaded; struct plist_head pushable_tasks; #endif + int rt_queued; + int rt_throttled; u64 rt_time; u64 rt_runtime; @@ -390,11 +454,45 @@ struct rt_rq { unsigned long rt_nr_boosted; struct rq *rq; - struct list_head leaf_rt_rq_list; struct task_group *tg; #endif }; +/* Deadline class' related fields in a runqueue */ +struct dl_rq { + /* runqueue is an rbtree, ordered by deadline */ + struct rb_root rb_root; + struct rb_node *rb_leftmost; + + unsigned long dl_nr_running; + +#ifdef CONFIG_SMP + /* + * Deadline values of the currently executing and the + * earliest ready task on this rq. Caching these facilitates + * the decision wether or not a ready but not running task + * should migrate somewhere else. + */ + struct { + u64 curr; + u64 next; + } earliest_dl; + + unsigned long dl_nr_migratory; + int overloaded; + + /* + * Tasks on this rq that can be pushed away. They are kept in + * an rb-tree, ordered by tasks' deadlines, with caching + * of the leftmost (earliest deadline) element. + */ + struct rb_root pushable_dl_tasks_root; + struct rb_node *pushable_dl_tasks_leftmost; +#else + struct dl_bw dl_bw; +#endif +}; + #ifdef CONFIG_SMP /* @@ -412,12 +510,30 @@ struct root_domain { cpumask_var_t span; cpumask_var_t online; + /* Indicate more than one runnable task for any CPU */ + bool overload; + + /* Indicate one or more cpus over-utilized (tipping point) */ + bool overutilized; + + /* + * The bit corresponding to a CPU gets set here if such CPU has more + * than one runnable -deadline task (as it is below for RT tasks). + */ + cpumask_var_t dlo_mask; + atomic_t dlo_count; + struct dl_bw dl_bw; + struct cpudl cpudl; + /* * The "RT overload" flag: it gets set if a CPU has more than * one runnable RT task. */ cpumask_var_t rto_mask; struct cpupri cpupri; + + /* Maximum cpu capacity in the system. */ + unsigned long max_cpu_capacity; }; extern struct root_domain def_root_domain; @@ -440,9 +556,14 @@ struct rq { * remote CPUs use both these fields when doing load calculation. */ unsigned int nr_running; +#ifdef CONFIG_NUMA_BALANCING + unsigned int nr_numa_running; + unsigned int nr_preferred_running; +#endif #define CPU_LOAD_IDX_MAX 5 unsigned long cpu_load[CPU_LOAD_IDX_MAX]; unsigned long last_load_update_tick; + unsigned int misfit_task; #ifdef CONFIG_NO_HZ_COMMON u64 nohz_stamp; unsigned long nohz_flags; @@ -459,18 +580,14 @@ struct rq { struct cfs_rq cfs; struct rt_rq rt; + struct dl_rq dl; #ifdef CONFIG_FAIR_GROUP_SCHED /* list of leaf cfs_rq on this cpu: */ struct list_head leaf_cfs_rq_list; -#ifdef CONFIG_SMP - unsigned long h_load_throttle; -#endif /* CONFIG_SMP */ -#endif /* CONFIG_FAIR_GROUP_SCHED */ -#ifdef CONFIG_RT_GROUP_SCHED - struct list_head leaf_rt_rq_list; -#endif + struct sched_avg avg; +#endif /* CONFIG_FAIR_GROUP_SCHED */ /* * This is part of a global counter where only the total sum @@ -493,14 +610,14 @@ struct rq { struct root_domain *rd; struct sched_domain *sd; - unsigned long cpu_power; + unsigned long cpu_capacity; + unsigned long cpu_capacity_orig; unsigned char idle_balance; /* For active balancing */ int post_schedule; int active_balance; int push_cpu; - struct task_struct *push_task; struct cpu_stop_work active_balance_work; /* cpu of this runqueue: */ int cpu; @@ -512,42 +629,9 @@ struct rq { u64 age_stamp; u64 idle_stamp; u64 avg_idle; - int cstate, wakeup_latency, wakeup_energy; -#endif -#ifdef CONFIG_SCHED_HMP - /* - * max_freq = user or thermal defined maximum - * max_possible_freq = maximum supported by hardware - */ - unsigned int cur_freq, max_freq, min_freq, max_possible_freq; - struct cpumask freq_domain_cpumask; - - struct hmp_sched_stats hmp_stats; - - int efficiency; /* Differentiate cpus with different IPC capability */ - int load_scale_factor; - int capacity; - int max_possible_capacity; - u64 window_start; - int prefer_idle; - u32 mostly_idle_load; - int mostly_idle_nr_run; - int mostly_idle_freq; - unsigned long hmp_flags; - - u64 cur_irqload; - u64 avg_irqload; - u64 irqload_ts; - -#ifdef CONFIG_SCHED_FREQ_INPUT - unsigned int old_busy_time; -#endif -#endif - -#ifdef CONFIG_SCHED_FREQ_INPUT - u64 curr_runnable_sum; - u64 prev_runnable_sum; + /* This is used to determine avg_idle's max value */ + u64 max_idle_balance_cost; #endif #ifdef CONFIG_IRQ_TIME_ACCOUNTING @@ -594,7 +678,11 @@ struct rq { struct llist_head wake_list; #endif - struct sched_avg avg; +#ifdef CONFIG_CPU_IDLE + /* Must be inspected within a rcu lock section */ + struct cpuidle_state *idle_state; + int idle_state_idx; +#endif }; static inline int cpu_of(struct rq *rq) @@ -606,16 +694,34 @@ static inline int cpu_of(struct rq *rq) #endif } -DECLARE_PER_CPU(struct rq, runqueues); +DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); #define cpu_rq(cpu) (&per_cpu(runqueues, (cpu))) -#define this_rq() (&__get_cpu_var(runqueues)) +#define this_rq() this_cpu_ptr(&runqueues) #define task_rq(p) cpu_rq(task_cpu(p)) #define cpu_curr(cpu) (cpu_rq(cpu)->curr) -#define raw_rq() (&__raw_get_cpu_var(runqueues)) +#define raw_rq() raw_cpu_ptr(&runqueues) + +static inline u64 rq_clock(struct rq *rq) +{ + return rq->clock; +} + +static inline u64 rq_clock_task(struct rq *rq) +{ + return rq->clock_task; +} + +#ifdef CONFIG_NUMA_BALANCING +extern void sched_setnuma(struct task_struct *p, int node); +extern int migrate_task_to(struct task_struct *p, int cpu); +extern int migrate_swap(struct task_struct *, struct task_struct *); +#endif /* CONFIG_NUMA_BALANCING */ #ifdef CONFIG_SMP +extern void sched_ttwu_pending(void); + #define rcu_dereference_check_sched_domain(p) \ rcu_dereference_check((p), \ lockdep_is_held(&sched_domains_mutex)) @@ -655,17 +761,36 @@ static inline struct sched_domain *highest_flag_domain(int cpu, int flag) return hsd; } +static inline struct sched_domain *lowest_flag_domain(int cpu, int flag) +{ + struct sched_domain *sd; + + for_each_domain(cpu, sd) { + if (sd->flags & flag) + break; + } + + return sd; +} + DECLARE_PER_CPU(struct sched_domain *, sd_llc); +DECLARE_PER_CPU(int, sd_llc_size); DECLARE_PER_CPU(int, sd_llc_id); +DECLARE_PER_CPU(struct sched_domain *, sd_numa); +DECLARE_PER_CPU(struct sched_domain *, sd_busy); +DECLARE_PER_CPU(struct sched_domain *, sd_asym); +DECLARE_PER_CPU(struct sched_domain *, sd_ea); -struct sched_group_power { +struct sched_group_capacity { atomic_t ref; /* - * CPU power of this group, SCHED_LOAD_SCALE being max power for a - * single CPU. + * CPU capacity of this group, SCHED_LOAD_SCALE being max capacity + * for a single CPU. */ - unsigned int power, power_orig; + unsigned long capacity; + unsigned long max_capacity; /* Max per-cpu capacity in group */ unsigned long next_update; + int imbalance; /* XXX unrelated to capacity but shared group state */ /* * Number of busy cpus in this group. */ @@ -679,7 +804,8 @@ struct sched_group { atomic_t ref; unsigned int group_weight; - struct sched_group_power *sgp; + struct sched_group_capacity *sgc; + struct sched_group_energy *sge; /* * The CPUs this group covers. @@ -702,7 +828,7 @@ static inline struct cpumask *sched_group_cpus(struct sched_group *sg) */ static inline struct cpumask *sched_group_mask(struct sched_group *sg) { - return to_cpumask(sg->sgp->cpumask); + return to_cpumask(sg->sgc->cpumask); } /** @@ -717,277 +843,55 @@ static inline unsigned int group_first_cpu(struct sched_group *group) extern int group_balance_cpu(struct sched_group *sg); /* - * Returns the rq capacity of any rq in a group. This does not play - * well with groups where rq capacity can change independently. + * Check that the per-cpu provided sd energy data is consistent for all cpus + * within the mask. */ -#define group_rq_capacity(group) capacity(cpu_rq(group_first_cpu(group))) - -#endif /* CONFIG_SMP */ - -#include "stats.h" -#include "auto_group.h" - -extern void init_new_task_load(struct task_struct *p); - -#ifdef CONFIG_SCHED_HMP - -#define WINDOW_STATS_RECENT 0 -#define WINDOW_STATS_MAX 1 -#define WINDOW_STATS_MAX_RECENT_AVG 2 -#define WINDOW_STATS_AVG 3 -#define WINDOW_STATS_INVALID_POLICY 4 - -extern struct mutex policy_mutex; -extern unsigned int sched_ravg_window; -extern unsigned int sched_use_pelt; -extern unsigned int sched_disable_window_stats; -extern unsigned int sched_enable_hmp; -extern unsigned int max_possible_freq; -extern unsigned int min_max_freq; -extern unsigned int pct_task_load(struct task_struct *p); -extern unsigned int max_possible_efficiency; -extern unsigned int min_possible_efficiency; -extern unsigned int max_capacity; -extern unsigned int min_capacity; -extern unsigned int max_load_scale_factor; -extern unsigned int max_possible_capacity; -extern cpumask_t mpc_mask; -extern unsigned long capacity_scale_cpu_efficiency(int cpu); -extern unsigned long capacity_scale_cpu_freq(int cpu); -extern unsigned int sched_mostly_idle_load; -extern unsigned int sched_small_task; -extern unsigned int sched_upmigrate; -extern unsigned int sched_downmigrate; -extern unsigned int sched_init_task_load_pelt; -extern unsigned int sched_init_task_load_windows; -extern unsigned int sched_heavy_task; - -extern void reset_cpu_hmp_stats(int cpu, int reset_cra); -extern void fixup_nr_big_small_task(int cpu, int reset_stats); -unsigned int max_task_load(void); -extern void sched_account_irqtime(int cpu, struct task_struct *curr, - u64 delta, u64 wallclock); -unsigned int cpu_temp(int cpu); -extern unsigned int nr_eligible_big_tasks(int cpu); - -/* - * 'load' is in reference to "best cpu" at its best frequency. - * Scale that in reference to a given cpu, accounting for how bad it is - * in reference to "best cpu". - */ -static inline u64 scale_load_to_cpu(u64 task_load, int cpu) +static inline void check_sched_energy_data(int cpu, sched_domain_energy_f fn, + const struct cpumask *cpumask) { - struct rq *rq = cpu_rq(cpu); + struct cpumask mask; + int i; - task_load *= (u64)rq->load_scale_factor; - task_load /= 1024; + cpumask_xor(&mask, cpumask, get_cpu_mask(cpu)); - return task_load; -} + for_each_cpu(i, &mask) { + int y; -static inline int capacity(struct rq *rq) -{ - return rq->capacity; -} - -static inline void -inc_cumulative_runnable_avg(struct hmp_sched_stats *stats, - struct task_struct *p) -{ - u32 task_load; + BUG_ON(fn(i)->nr_idle_states != fn(cpu)->nr_idle_states); - if (!sched_enable_hmp || sched_disable_window_stats) - return; - - task_load = sched_use_pelt ? p->se.avg.runnable_avg_sum_scaled : - (sched_disable_window_stats ? 0 : p->ravg.demand); - - stats->cumulative_runnable_avg += task_load; -} - -static inline void -dec_cumulative_runnable_avg(struct hmp_sched_stats *stats, - struct task_struct *p) -{ - u32 task_load; - - if (!sched_enable_hmp || sched_disable_window_stats) - return; - - task_load = sched_use_pelt ? p->se.avg.runnable_avg_sum_scaled : - (sched_disable_window_stats ? 0 : p->ravg.demand); - - stats->cumulative_runnable_avg -= task_load; - - BUG_ON((s64)stats->cumulative_runnable_avg < 0); -} - -#define pct_to_real(tunable) \ - (div64_u64((u64)tunable * (u64)max_task_load(), 100)) - -#define real_to_pct(tunable) \ - (div64_u64((u64)tunable * (u64)100, (u64)max_task_load())) - -#define SCHED_HIGH_IRQ_TIMEOUT 3 -static inline u64 sched_irqload(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - s64 delta; - - delta = get_jiffies_64() - rq->irqload_ts; - /* - * Current context can be preempted by irq and rq->irqload_ts can be - * updated by irq context so that delta can be negative. - * But this is okay and we can safely return as this means there - * was recent irq occurrence. - */ - - if (delta < SCHED_HIGH_IRQ_TIMEOUT) - return rq->avg_irqload; - else - return 0; -} - -static inline int sched_cpu_high_irqload(int cpu) -{ - return sched_irqload(cpu) >= sysctl_sched_cpu_high_irqload; -} - -#else /* CONFIG_SCHED_HMP */ - -struct hmp_sched_stats; - -static inline unsigned int nr_eligible_big_tasks(int cpu) -{ - return 0; -} - -static inline int pct_task_load(struct task_struct *p) { return 0; } - -static inline int capacity(struct rq *rq) -{ - return SCHED_LOAD_SCALE; -} - -static inline void inc_cumulative_runnable_avg(struct hmp_sched_stats *stats, - struct task_struct *p) -{ -} - -static inline void dec_cumulative_runnable_avg(struct hmp_sched_stats *stats, - struct task_struct *p) -{ -} - -static inline unsigned long capacity_scale_cpu_efficiency(int cpu) -{ - return SCHED_LOAD_SCALE; -} - -static inline unsigned long capacity_scale_cpu_freq(int cpu) -{ - return SCHED_LOAD_SCALE; -} - -static inline void sched_account_irqtime(int cpu, struct task_struct *curr, - u64 delta, u64 wallclock) -{ -} - -static inline int sched_cpu_high_irqload(int cpu) { return 0; } - -#endif /* CONFIG_SCHED_HMP */ - -#ifdef CONFIG_SCHED_FREQ_INPUT -extern void check_for_freq_change(struct rq *rq); - -/* Is frequency of two cpus synchronized with each other? */ -static inline int same_freq_domain(int src_cpu, int dst_cpu) -{ - struct rq *rq = cpu_rq(src_cpu); - - if (src_cpu == dst_cpu) - return 1; - - return cpumask_test_cpu(dst_cpu, &rq->freq_domain_cpumask); -} - -#else /* CONFIG_SCHED_FREQ_INPUT */ - -#define sched_migration_fixup 0 - -static inline void check_for_freq_change(struct rq *rq) { } - -static inline int same_freq_domain(int src_cpu, int dst_cpu) -{ - return 1; -} - -#endif /* CONFIG_SCHED_FREQ_INPUT */ - -#ifdef CONFIG_SCHED_HMP - -#define BOOST_KICK 0 -#define CPU_RESERVED 1 - -static inline int is_reserved(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - return test_bit(CPU_RESERVED, &rq->hmp_flags); -} - -static inline int mark_reserved(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - /* Name boost_flags as hmp_flags? */ - return test_and_set_bit(CPU_RESERVED, &rq->hmp_flags); -} + for (y = 0; y < (fn(i)->nr_idle_states); y++) { + BUG_ON(fn(i)->idle_states[y].power != + fn(cpu)->idle_states[y].power); + } -static inline void clear_reserved(int cpu) -{ - struct rq *rq = cpu_rq(cpu); + BUG_ON(fn(i)->nr_cap_states != fn(cpu)->nr_cap_states); - clear_bit(CPU_RESERVED, &rq->hmp_flags); + for (y = 0; y < (fn(i)->nr_cap_states); y++) { + BUG_ON(fn(i)->cap_states[y].cap != + fn(cpu)->cap_states[y].cap); + BUG_ON(fn(i)->cap_states[y].power != + fn(cpu)->cap_states[y].power); + } + } } -int mostly_idle_cpu(int cpu); -extern void check_for_migration(struct rq *rq, struct task_struct *p); -extern void pre_big_small_task_count_change(const struct cpumask *cpus); -extern void post_big_small_task_count_change(const struct cpumask *cpus); -extern void set_hmp_defaults(void); -extern unsigned int power_cost_at_freq(int cpu, unsigned int freq); -extern void reset_all_window_stats(u64 window_start, unsigned int window_size); -extern void boost_kick(int cpu); -extern int sched_boost(void); - -#else /* CONFIG_SCHED_HMP */ - -#define sched_enable_hmp 0 -#define sched_freq_legacy_mode 1 - -static inline void check_for_migration(struct rq *rq, struct task_struct *p) { } -static inline void pre_big_small_task_count_change(void) { } -static inline void post_big_small_task_count_change(void) { } -static inline void set_hmp_defaults(void) { } - -static inline void clear_reserved(int cpu) { } +#else -#define power_cost_at_freq(...) 0 +static inline void sched_ttwu_pending(void) { } -#define trace_sched_cpu_load(...) +#endif /* CONFIG_SMP */ -#endif /* CONFIG_SCHED_HMP */ +#include "stats.h" +#include "auto_group.h" #ifdef CONFIG_CGROUP_SCHED /* * Return the group to which this tasks belongs. * - * We cannot use task_subsys_state() and friends because the cgroup - * subsystem changes that value before the cgroup_subsys::attach() method - * is called, therefore we cannot pin it and might observe the wrong value. + * We cannot use task_css() and friends because the cgroup subsystem + * changes that value before the cgroup_subsys::attach() method is called, + * therefore we cannot pin it and might observe the wrong value. * * The same is true for autogroup's p->signal->autogroup->tg, the autogroup * core changes this before calling sched_move_task(). @@ -1000,11 +904,6 @@ static inline struct task_group *task_group(struct task_struct *p) return p->sched_task_group; } -static inline bool task_notify_on_migrate(struct task_struct *p) -{ - return task_group(p)->notify_on_migrate; -} - /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { @@ -1030,10 +929,7 @@ static inline struct task_group *task_group(struct task_struct *p) { return NULL; } -static inline bool task_notify_on_migrate(struct task_struct *p) -{ - return false; -} + #endif /* CONFIG_CGROUP_SCHED */ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) @@ -1047,6 +943,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) */ smp_wmb(); task_thread_info(p)->cpu = cpu; + p->wake_cpu = cpu; #endif } @@ -1073,20 +970,10 @@ enum { #undef SCHED_FEAT #if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL) -static __always_inline bool static_branch__true(struct static_key *key) -{ - return static_key_true(key); /* Not out of line branch. */ -} - -static __always_inline bool static_branch__false(struct static_key *key) -{ - return static_key_false(key); /* Out of line branch. */ -} - #define SCHED_FEAT(name, enabled) \ static __always_inline bool static_branch_##name(struct static_key *key) \ { \ - return static_branch__##enabled(key); \ + return static_key_##enabled(key); \ } #include "features.h" @@ -1124,8 +1011,6 @@ static inline u64 global_rt_runtime(void) return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC; } - - static inline int task_current(struct rq *rq, struct task_struct *p) { return rq->curr == p; @@ -1140,6 +1025,15 @@ static inline int task_running(struct rq *rq, struct task_struct *p) #endif } +static inline int task_on_rq_queued(struct task_struct *p) +{ + return p->on_rq == TASK_ON_RQ_QUEUED; +} + +static inline int task_on_rq_migrating(struct task_struct *p) +{ + return p->on_rq == TASK_ON_RQ_MIGRATING; +} #ifndef prepare_arch_switch # define prepare_arch_switch(next) do { } while (0) @@ -1151,7 +1045,6 @@ static inline int task_running(struct rq *rq, struct task_struct *p) # define finish_arch_post_lock_switch() do { } while (0) #endif -#ifndef __ARCH_WANT_UNLOCKED_CTXSW static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next) { #ifdef CONFIG_SMP @@ -1189,35 +1082,6 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) raw_spin_unlock_irq(&rq->lock); } -#else /* __ARCH_WANT_UNLOCKED_CTXSW */ -static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next) -{ -#ifdef CONFIG_SMP - /* - * We can optimise this out completely for !SMP, because the - * SMP rebalancing from interrupt is the only thing that cares - * here. - */ - next->on_cpu = 1; -#endif - raw_spin_unlock(&rq->lock); -} - -static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) -{ -#ifdef CONFIG_SMP - /* - * After ->on_cpu is cleared, the task can be moved to a different CPU. - * We must ensure this doesn't happen until the switch is completely - * finished. - */ - smp_wmb(); - prev->on_cpu = 0; -#endif - local_irq_enable(); -} -#endif /* __ARCH_WANT_UNLOCKED_CTXSW */ - /* * wake flags */ @@ -1225,24 +1089,6 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) #define WF_FORK 0x02 /* child wakeup after fork */ #define WF_MIGRATED 0x4 /* internal use, task got migrated */ -static inline void update_load_add(struct load_weight *lw, unsigned long inc) -{ - lw->weight += inc; - lw->inv_weight = 0; -} - -static inline void update_load_sub(struct load_weight *lw, unsigned long dec) -{ - lw->weight -= dec; - lw->inv_weight = 0; -} - -static inline void update_load_set(struct load_weight *lw, unsigned long w) -{ - lw->weight = w; - lw->inv_weight = 0; -} - /* * To aid in avoiding the subversion of "niceness" due to uneven distribution * of tasks with abnormal "nice" values across CPUs the contribution that @@ -1303,9 +1149,13 @@ static const u32 prio_to_wmult[40] = { #else #define ENQUEUE_WAKING 0 #endif +#define ENQUEUE_REPLENISH 8 +#define ENQUEUE_WAKEUP_NEW 16 #define DEQUEUE_SLEEP 1 +#define RETRY_TASK ((void *)-1UL) + struct sched_class { const struct sched_class *next; @@ -1316,14 +1166,22 @@ struct sched_class { void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int flags); - struct task_struct * (*pick_next_task) (struct rq *rq); + /* + * It is the responsibility of the pick_next_task() method that will + * return the next task to call put_prev_task() on the @prev task or + * something equivalent. + * + * May return RETRY_TASK when it finds a higher prio class has runnable + * tasks. + */ + struct task_struct * (*pick_next_task) (struct rq *rq, + struct task_struct *prev); void (*put_prev_task) (struct rq *rq, struct task_struct *p); #ifdef CONFIG_SMP - int (*select_task_rq)(struct task_struct *p, int sd_flag, int flags); + int (*select_task_rq)(struct task_struct *p, int task_cpu, int sd_flag, int flags); void (*migrate_task_rq)(struct task_struct *p, int next_cpu); - void (*pre_schedule) (struct rq *this_rq, struct task_struct *task); void (*post_schedule) (struct rq *this_rq); void (*task_waking) (struct task_struct *task); void (*task_woken) (struct rq *this_rq, struct task_struct *task); @@ -1338,6 +1196,7 @@ struct sched_class { void (*set_curr_task) (struct rq *rq); void (*task_tick) (struct rq *rq, struct task_struct *p, int queued); void (*task_fork) (struct task_struct *p); + void (*task_dead) (struct task_struct *p); void (*switched_from) (struct rq *this_rq, struct task_struct *task); void (*switched_to) (struct rq *this_rq, struct task_struct *task); @@ -1347,20 +1206,24 @@ struct sched_class { unsigned int (*get_rr_interval) (struct rq *rq, struct task_struct *task); + void (*update_curr) (struct rq *rq); + #ifdef CONFIG_FAIR_GROUP_SCHED void (*task_move_group) (struct task_struct *p, int on_rq); #endif -#ifdef CONFIG_SCHED_HMP - void (*inc_hmp_sched_stats)(struct rq *rq, struct task_struct *p); - void (*dec_hmp_sched_stats)(struct rq *rq, struct task_struct *p); -#endif }; +static inline void put_prev_task(struct rq *rq, struct task_struct *prev) +{ + prev->sched_class->put_prev_task(rq, prev); +} + #define sched_class_highest (&stop_sched_class) #define for_each_class(class) \ for (class = sched_class_highest; class; class = class->next) extern const struct sched_class stop_sched_class; +extern const struct sched_class dl_sched_class; extern const struct sched_class rt_sched_class; extern const struct sched_class fair_sched_class; extern const struct sched_class idle_sched_class; @@ -1368,77 +1231,149 @@ extern const struct sched_class idle_sched_class; #ifdef CONFIG_SMP -extern void update_group_power(struct sched_domain *sd, int cpu); +extern void update_group_capacity(struct sched_domain *sd, int cpu); -extern void trigger_load_balance(struct rq *rq, int cpu); -extern void idle_balance(int this_cpu, struct rq *this_rq); +extern void trigger_load_balance(struct rq *rq); -/* - * Only depends on SMP, FAIR_GROUP_SCHED may be removed when runnable_avg - * becomes useful in lb - */ -#if defined(CONFIG_FAIR_GROUP_SCHED) extern void idle_enter_fair(struct rq *this_rq); extern void idle_exit_fair(struct rq *this_rq); + #else -static inline void idle_enter_fair(struct rq *this_rq) {} -static inline void idle_exit_fair(struct rq *this_rq) {} + +static inline void idle_enter_fair(struct rq *rq) { } +static inline void idle_exit_fair(struct rq *rq) { } + #endif -#else /* CONFIG_SMP */ +#ifdef CONFIG_CPU_IDLE +static inline void idle_set_state(struct rq *rq, + struct cpuidle_state *idle_state) +{ + rq->idle_state = idle_state; +} -static inline void idle_balance(int cpu, struct rq *rq) +static inline struct cpuidle_state *idle_get_state(struct rq *rq) { + WARN_ON(!rcu_read_lock_held()); + return rq->idle_state; } +static inline void idle_set_state_idx(struct rq *rq, int idle_state_idx) +{ + rq->idle_state_idx = idle_state_idx; +} + +static inline int idle_get_state_idx(struct rq *rq) +{ + WARN_ON(!rcu_read_lock_held()); + return rq->idle_state_idx; +} +#else +static inline void idle_set_state(struct rq *rq, + struct cpuidle_state *idle_state) +{ +} + +static inline struct cpuidle_state *idle_get_state(struct rq *rq) +{ + return NULL; +} + +static inline void idle_set_state_idx(struct rq *rq, int idle_state_idx) +{ +} + +static inline int idle_get_state_idx(struct rq *rq) +{ + return -1; +} #endif -#ifdef CONFIG_SYSRQ_SCHED_DEBUG -extern void sysrq_sched_debug_show(void); +#ifndef arch_scale_cpu_capacity +static __always_inline +unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu) +{ + if (sd && (sd->flags & SD_SHARE_CPUCAPACITY) && (sd->span_weight > 1)) + return sd->smt_gain / sd->span_weight; + + return SCHED_CAPACITY_SCALE; +} #endif + +unsigned long capacity_orig_of(int cpu); + +extern struct static_key __sched_energy_freq; +static inline bool sched_energy_freq(void) +{ + return static_key_false(&__sched_energy_freq); +} + +#ifdef CONFIG_CPU_FREQ_GOV_SCHED +void cpufreq_sched_set_cap(int cpu, unsigned long util); +void cpufreq_sched_reset_cap(int cpu); +#else +static inline void cpufreq_sched_set_cap(int cpu, unsigned long util) +{ } +static inline void cpufreq_sched_reset_cap(int cpu) +{ } +#endif + +extern void sysrq_sched_debug_show(void); extern void sched_init_granularity(void); extern void update_max_interval(void); + +extern void init_sched_dl_class(void); extern void init_sched_rt_class(void); extern void init_sched_fair_class(void); +extern void init_sched_dl_class(void); -extern void resched_task(struct task_struct *p); +extern void resched_curr(struct rq *rq); extern void resched_cpu(int cpu); extern struct rt_bandwidth def_rt_bandwidth; extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime); +extern struct dl_bandwidth def_dl_bandwidth; +extern void init_dl_bandwidth(struct dl_bandwidth *dl_b, u64 period, u64 runtime); +extern void init_dl_task_timer(struct sched_dl_entity *dl_se); + +unsigned long to_ratio(u64 period, u64 runtime); + extern void update_idle_cpu_load(struct rq *this_rq); -#ifdef CONFIG_PARAVIRT -static inline u64 steal_ticks(u64 steal) +extern void init_task_runnable_average(struct task_struct *p); + +static inline void add_nr_running(struct rq *rq, unsigned count) { - if (unlikely(steal > NSEC_PER_SEC)) - return div_u64(steal, TICK_NSEC); + unsigned prev_nr = rq->nr_running; - return __iter_div_u64_rem(steal, TICK_NSEC, &steal); -} -#endif + rq->nr_running = prev_nr + count; -static inline void inc_nr_running(struct rq *rq) -{ - sched_update_nr_prod(cpu_of(rq), 1, true); - rq->nr_running++; + if (prev_nr < 2 && rq->nr_running >= 2) { +#ifdef CONFIG_SMP + if (!rq->rd->overload) + rq->rd->overload = true; +#endif #ifdef CONFIG_NO_HZ_FULL - if (rq->nr_running == 2) { if (tick_nohz_full_cpu(rq->cpu)) { - /* Order rq->nr_running write against the IPI */ - smp_wmb(); - smp_send_reschedule(rq->cpu); + /* + * Tick is needed if more than one task runs on a CPU. + * Send the target an IPI to kick it out of nohz mode. + * + * We assume that IPI implies full memory barrier and the + * new value of rq->nr_running is visible on reception + * from the target. + */ + tick_nohz_full_kick_cpu(rq->cpu); } - } #endif + } } -static inline void dec_nr_running(struct rq *rq) +static inline void sub_nr_running(struct rq *rq, unsigned count) { - sched_update_nr_prod(cpu_of(rq), 1, false); - rq->nr_running--; + rq->nr_running -= count; } static inline void rq_last_tick_reset(struct rq *rq) @@ -1493,14 +1428,18 @@ static inline int hrtick_enabled(struct rq *rq) #ifdef CONFIG_SMP extern void sched_avg_update(struct rq *rq); +extern unsigned long arch_scale_freq_capacity(struct sched_domain *sd, int cpu); +extern unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu); static inline void sched_rt_avg_update(struct rq *rq, u64 rt_delta) { - rq->rt_avg += rt_delta; + rq->rt_avg += rt_delta * arch_scale_freq_capacity(NULL, cpu_of(rq)); sched_avg_update(rq); } + #else static inline void sched_rt_avg_update(struct rq *rq, u64 rt_delta) { } static inline void sched_avg_update(struct rq *rq) { } +static inline void gov_cfs_update_cpu(int cpu) {} #endif extern void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period); @@ -1581,6 +1520,33 @@ static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest) lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_); } +static inline void double_lock(spinlock_t *l1, spinlock_t *l2) +{ + if (l1 > l2) + swap(l1, l2); + + spin_lock(l1); + spin_lock_nested(l2, SINGLE_DEPTH_NESTING); +} + +static inline void double_lock_irq(spinlock_t *l1, spinlock_t *l2) +{ + if (l1 > l2) + swap(l1, l2); + + spin_lock_irq(l1); + spin_lock_nested(l2, SINGLE_DEPTH_NESTING); +} + +static inline void double_raw_lock(raw_spinlock_t *l1, raw_spinlock_t *l2) +{ + if (l1 > l2) + swap(l1, l2); + + raw_spin_lock(l1); + raw_spin_lock_nested(l2, SINGLE_DEPTH_NESTING); +} + /* * double_rq_lock - safely lock two runqueues * @@ -1665,6 +1631,7 @@ extern void print_rt_stats(struct seq_file *m, int cpu); extern void init_cfs_rq(struct cfs_rq *cfs_rq); extern void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq); +extern void init_dl_rq(struct dl_rq *dl_rq, struct rq *rq); extern void cfs_bandwidth_usage_inc(void); extern void cfs_bandwidth_usage_dec(void); @@ -1678,9 +1645,6 @@ enum rq_nohz_flag_bits { #define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags) #endif -#define NOHZ_KICK_ANY 0 -#define NOHZ_KICK_RESTRICT 1 - #ifdef CONFIG_IRQ_TIME_ACCOUNTING DECLARE_PER_CPU(u64, cpu_hardirq_time); diff --git a/kernel/sched/stats.c b/kernel/sched/stats.c index da98af347e8b6..a476bea17fbc6 100644 --- a/kernel/sched/stats.c +++ b/kernel/sched/stats.c @@ -142,4 +142,4 @@ static int __init proc_schedstat_init(void) proc_create("schedstat", 0, NULL, &proc_schedstat_operations); return 0; } -module_init(proc_schedstat_init); +subsys_initcall(proc_schedstat_init); diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h index 2ef90a51ec5e3..4ab7043396569 100644 --- a/kernel/sched/stats.h +++ b/kernel/sched/stats.h @@ -59,9 +59,9 @@ static inline void sched_info_reset_dequeued(struct task_struct *t) * from dequeue_task() to account for possible rq->clock skew across cpus. The * delta taken on each cpu would annul the skew. */ -static inline void sched_info_dequeued(struct task_struct *t) +static inline void sched_info_dequeued(struct rq *rq, struct task_struct *t) { - unsigned long long now = task_rq(t)->clock, delta = 0; + unsigned long long now = rq_clock(rq), delta = 0; if (unlikely(sched_info_on())) if (t->sched_info.last_queued) @@ -69,7 +69,7 @@ static inline void sched_info_dequeued(struct task_struct *t) sched_info_reset_dequeued(t); t->sched_info.run_delay += delta; - rq_sched_info_dequeued(task_rq(t), delta); + rq_sched_info_dequeued(rq, delta); } /* @@ -77,9 +77,9 @@ static inline void sched_info_dequeued(struct task_struct *t) * long it was waiting to run. We also note when it began so that we * can keep stats on how long its timeslice is. */ -static void sched_info_arrive(struct task_struct *t) +static void sched_info_arrive(struct rq *rq, struct task_struct *t) { - unsigned long long now = task_rq(t)->clock, delta = 0; + unsigned long long now = rq_clock(rq), delta = 0; if (t->sched_info.last_queued) delta = now - t->sched_info.last_queued; @@ -88,7 +88,7 @@ static void sched_info_arrive(struct task_struct *t) t->sched_info.last_arrival = now; t->sched_info.pcount++; - rq_sched_info_arrive(task_rq(t), delta); + rq_sched_info_arrive(rq, delta); } /* @@ -96,29 +96,30 @@ static void sched_info_arrive(struct task_struct *t) * the timestamp if it is already not set. It's assumed that * sched_info_dequeued() will clear that stamp when appropriate. */ -static inline void sched_info_queued(struct task_struct *t) +static inline void sched_info_queued(struct rq *rq, struct task_struct *t) { if (unlikely(sched_info_on())) if (!t->sched_info.last_queued) - t->sched_info.last_queued = task_rq(t)->clock; + t->sched_info.last_queued = rq_clock(rq); } /* - * Called when a process ceases being the active-running process, either - * voluntarily or involuntarily. Now we can calculate how long we ran. + * Called when a process ceases being the active-running process involuntarily + * due, typically, to expiring its time slice (this may also be called when + * switching to the idle task). Now we can calculate how long we ran. * Also, if the process is still in the TASK_RUNNING state, call * sched_info_queued() to mark that it has now again started waiting on * the runqueue. */ -static inline void sched_info_depart(struct task_struct *t) +static inline void sched_info_depart(struct rq *rq, struct task_struct *t) { - unsigned long long delta = task_rq(t)->clock - + unsigned long long delta = rq_clock(rq) - t->sched_info.last_arrival; - rq_sched_info_depart(task_rq(t), delta); + rq_sched_info_depart(rq, delta); if (t->state == TASK_RUNNING) - sched_info_queued(t); + sched_info_queued(rq, t); } /* @@ -127,32 +128,34 @@ static inline void sched_info_depart(struct task_struct *t) * the idle task.) We are only called when prev != next. */ static inline void -__sched_info_switch(struct task_struct *prev, struct task_struct *next) +__sched_info_switch(struct rq *rq, + struct task_struct *prev, struct task_struct *next) { - struct rq *rq = task_rq(prev); - /* * prev now departs the cpu. It's not interesting to record * stats about how efficient we were at scheduling the idle * process, however. */ if (prev != rq->idle) - sched_info_depart(prev); + sched_info_depart(rq, prev); if (next != rq->idle) - sched_info_arrive(next); + sched_info_arrive(rq, next); } static inline void -sched_info_switch(struct task_struct *prev, struct task_struct *next) +sched_info_switch(struct rq *rq, + struct task_struct *prev, struct task_struct *next) { if (unlikely(sched_info_on())) - __sched_info_switch(prev, next); + __sched_info_switch(rq, prev, next); } #else -#define sched_info_queued(t) do { } while (0) +#define sched_info_queued(rq, t) do { } while (0) #define sched_info_reset_dequeued(t) do { } while (0) -#define sched_info_dequeued(t) do { } while (0) -#define sched_info_switch(t, next) do { } while (0) +#define sched_info_dequeued(rq, t) do { } while (0) +#define sched_info_depart(rq, t) do { } while (0) +#define sched_info_arrive(rq, next) do { } while (0) +#define sched_info_switch(rq, t, next) do { } while (0) #endif /* CONFIG_SCHEDSTATS || CONFIG_TASK_DELAY_ACCT */ /* @@ -161,6 +164,39 @@ sched_info_switch(struct task_struct *prev, struct task_struct *next) * on CONFIG_SCHEDSTATS. */ +/** + * cputimer_running - return true if cputimer is running + * + * @tsk: Pointer to target task. + */ +static inline bool cputimer_running(struct task_struct *tsk) + +{ + struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; + + if (!cputimer->running) + return false; + + /* + * After we flush the task's sum_exec_runtime to sig->sum_sched_runtime + * in __exit_signal(), we won't account to the signal struct further + * cputime consumed by that task, even though the task can still be + * ticking after __exit_signal(). + * + * In order to keep a consistent behaviour between thread group cputime + * and thread group cputimer accounting, lets also ignore the cputime + * elapsing after __exit_signal() in any thread group timer running. + * + * This makes sure that POSIX CPU clocks and timers are synchronized, so + * that a POSIX CPU timer won't expire while the corresponding POSIX CPU + * clock delta is behind the expiring timer value. + */ + if (unlikely(!tsk->sighand)) + return false; + + return true; +} + /** * account_group_user_time - Maintain utime for a thread group. * @@ -176,7 +212,7 @@ static inline void account_group_user_time(struct task_struct *tsk, { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; - if (!cputimer->running) + if (!cputimer_running(tsk)) return; raw_spin_lock(&cputimer->lock); @@ -199,7 +235,7 @@ static inline void account_group_system_time(struct task_struct *tsk, { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; - if (!cputimer->running) + if (!cputimer_running(tsk)) return; raw_spin_lock(&cputimer->lock); @@ -222,7 +258,7 @@ static inline void account_group_exec_runtime(struct task_struct *tsk, { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; - if (!cputimer->running) + if (!cputimer_running(tsk)) return; raw_spin_lock(&cputimer->lock); diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c index d25c5f3f07b74..79ffec45a6acd 100644 --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -11,66 +11,43 @@ #ifdef CONFIG_SMP static int -select_task_rq_stop(struct task_struct *p, int sd_flag, int flags) +select_task_rq_stop(struct task_struct *p, int cpu, int sd_flag, int flags) { return task_cpu(p); /* stop tasks as never migrate */ } #endif /* CONFIG_SMP */ -#ifdef CONFIG_SCHED_HMP - -static void -inc_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p) -{ - inc_cumulative_runnable_avg(&rq->hmp_stats, p); -} - -static void -dec_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p) -{ - dec_cumulative_runnable_avg(&rq->hmp_stats, p); -} - -#else /* CONFIG_SCHED_HMP */ - -static inline void -inc_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p) { } - -static inline void -dec_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p) { } - -#endif /* CONFIG_SCHED_HMP */ - static void check_preempt_curr_stop(struct rq *rq, struct task_struct *p, int flags) { /* we're never preempted */ } -static struct task_struct *pick_next_task_stop(struct rq *rq) +static struct task_struct * +pick_next_task_stop(struct rq *rq, struct task_struct *prev) { struct task_struct *stop = rq->stop; - if (stop && stop->on_rq) { - stop->se.exec_start = rq->clock_task; - return stop; - } + if (!stop || !task_on_rq_queued(stop)) + return NULL; + + put_prev_task(rq, prev); + + stop->se.exec_start = rq_clock_task(rq); - return NULL; + return stop; } static void enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags) { - inc_nr_running(rq); - inc_hmp_sched_stats_stop(rq, p); + add_nr_running(rq, 1); } static void dequeue_task_stop(struct rq *rq, struct task_struct *p, int flags) { - dec_nr_running(rq); - dec_hmp_sched_stats_stop(rq, p); + sub_nr_running(rq, 1); } static void yield_task_stop(struct rq *rq) @@ -83,7 +60,7 @@ static void put_prev_task_stop(struct rq *rq, struct task_struct *prev) struct task_struct *curr = rq->curr; u64 delta_exec; - delta_exec = rq->clock_task - curr->se.exec_start; + delta_exec = rq_clock_task(rq) - curr->se.exec_start; if (unlikely((s64)delta_exec < 0)) delta_exec = 0; @@ -93,7 +70,7 @@ static void put_prev_task_stop(struct rq *rq, struct task_struct *prev) curr->se.sum_exec_runtime += delta_exec; account_group_exec_runtime(curr, delta_exec); - curr->se.exec_start = rq->clock_task; + curr->se.exec_start = rq_clock_task(rq); cpuacct_charge(curr, delta_exec); } @@ -105,7 +82,7 @@ static void set_curr_task_stop(struct rq *rq) { struct task_struct *stop = rq->stop; - stop->se.exec_start = rq->clock_task; + stop->se.exec_start = rq_clock_task(rq); } static void switched_to_stop(struct rq *rq, struct task_struct *p) @@ -125,11 +102,15 @@ get_rr_interval_stop(struct rq *rq, struct task_struct *task) return 0; } +static void update_curr_stop(struct rq *rq) +{ +} + /* * Simple, special scheduling class for the per-CPU stop tasks: */ const struct sched_class stop_sched_class = { - .next = &rt_sched_class, + .next = &dl_sched_class, .enqueue_task = enqueue_task_stop, .dequeue_task = dequeue_task_stop, @@ -151,8 +132,5 @@ const struct sched_class stop_sched_class = { .prio_changed = prio_changed_stop, .switched_to = switched_to_stop, -#ifdef CONFIG_SCHED_HMP - .inc_hmp_sched_stats = inc_hmp_sched_stats_stop, - .dec_hmp_sched_stats = dec_hmp_sched_stats_stop, -#endif + .update_curr = update_curr_stop, }; diff --git a/kernel/sched/tune.c b/kernel/sched/tune.c new file mode 100644 index 0000000000000..779bb0f658535 --- /dev/null +++ b/kernel/sched/tune.c @@ -0,0 +1,720 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "sched.h" +#include "tune.h" + +unsigned int sysctl_sched_cfs_boost __read_mostly = 0; + +/* Performance Boost region (B) threshold params */ +static int perf_boost_idx; + +/* Performance Constraint region (C) threshold params */ +static int perf_constrain_idx; + +/** + * Performance-Energy (P-E) Space thresholds constants + */ +struct threshold_params { + int nrg_gain; + int cap_gain; +}; + +/* + * System specific P-E space thresholds constants + */ +static struct threshold_params +threshold_gains[] = { + { 0, 4 }, /* >= 0% */ + { 1, 4 }, /* >= 10% */ + { 2, 4 }, /* >= 20% */ + { 3, 4 }, /* >= 30% */ + { 4, 4 }, /* >= 40% */ + { 4, 3 }, /* >= 50% */ + { 4, 2 }, /* >= 60% */ + { 4, 1 }, /* >= 70% */ + { 4, 0 }, /* >= 80% */ + { 4, 0 } /* >= 90% */ +}; + +/* + * System energy normalization constants + */ +struct target_nrg { + unsigned long min_power; + unsigned long max_power; + unsigned long nrg_shift; + unsigned long nrg_mult; +}; + +/* + * Target specific system energy normalization constants + * NOTE: These values are specific for ARM TC2 and they are derived from the + * energy model data defined in: arch/arm/kernel/topology.c + */ +static struct target_nrg +schedtune_target_nrg = { + + /* + * TC2 Min CPUs power: + * all CPUs idle, all clusters in deep idle: + * 0 * 3 + 0 * 2 + 10 + 25 + */ + .min_power = 35, + + /* + * TC2 Max CPUs power: + * all CPUs fully utilized while running at max OPP: + * 1024 * 3 + 6997 * 2 + 4905 + 15200 + */ + + .max_power = 37171, + + /* + * Fast integer division by constant: + * Constant : Max - Min (C) = 37171 - 35 = 37136 + * Precision : 0.1% (P) = 0.1 + * Reference : C * 100 / P (R) = 3713600 + * + * Thus: + * Shift bifs : ceil(log(R,2)) (S) = 26 + * Mult const : round(2^S/C) (M) = 1807 + * + * This allows to compute the normalized energy: + * system_energy / C + * as: + * (system_energy * M) >> S + */ + .nrg_shift = 26, /* S */ + .nrg_mult = 1807, /* M */ +}; + +/* + * System energy normalization + * Returns the normalized value, in the range [0..SCHED_LOAD_SCALE], + * corresponding to the specified energy variation. + */ +int +schedtune_normalize_energy(int energy_diff) +{ + long long normalized_nrg = energy_diff; + int max_delta; + + /* Check for boundaries */ + max_delta = schedtune_target_nrg.max_power; + max_delta -= schedtune_target_nrg.min_power; + WARN_ON(abs(energy_diff) >= max_delta); + + /* Scale by energy magnitude */ + normalized_nrg <<= SCHED_LOAD_SHIFT; + + /* Normalize on max energy for target platform */ + normalized_nrg *= schedtune_target_nrg.nrg_mult; + normalized_nrg >>= schedtune_target_nrg.nrg_shift; + + return normalized_nrg; +} + +static int +__schedtune_accept_deltas(int nrg_delta, int cap_delta, + int perf_boost_idx, int perf_constrain_idx) { + int energy_payoff; + + /* Performance Boost (B) region */ + if (nrg_delta > 0 && cap_delta > 0) { + /* + * energy_payoff criteria: + * cap_delta / nrg_delta > cap_gain / nrg_gain + * which is: + * nrg_delta * cap_gain < cap_delta * nrg_gain + */ + energy_payoff = cap_delta * threshold_gains[perf_boost_idx].nrg_gain; + energy_payoff -= nrg_delta * threshold_gains[perf_boost_idx].cap_gain; +#if 0 + // TJK: fix tracing + trace_sched_tune_filter( + threshold_gains[perf_boost_idx].nrg_gain, + threshold_gains[perf_boost_idx].cap_gain, + energy_payoff, 8); +#endif + return energy_payoff; + } + + /* Performance Constraint (C) region */ + if (nrg_delta < 0 && cap_delta < 0) { + /* + * energy_payoff criteria: + * cap_delta / nrg_delta < cap_gain / nrg_gain + * which is: + * nrg_delta * cap_gain > cap_delta * nrg_gain + */ + energy_payoff = nrg_delta * threshold_gains[perf_constrain_idx].cap_gain; + energy_payoff -= cap_delta * threshold_gains[perf_constrain_idx].nrg_gain; + +#if 0 + // TJK: fix tracing + trace_sched_tune_filter( + threshold_gains[perf_constrain_idx].nrg_gain, + threshold_gains[perf_constrain_idx].cap_gain, + energy_payoff, 6); +#endif + return energy_payoff; + } + + /* Default: reject schedule candidate */ + return -INT_MAX; +} + +#ifdef CONFIG_CGROUP_SCHEDTUNE + +/* + * EAS scheduler tunables for task groups. + */ + +/* SchdTune tunables for a group of tasks */ +struct schedtune { + /* SchedTune CGroup subsystem */ + struct cgroup_subsys_state css; + + /* Boost group allocated ID */ + int idx; + + /* Boost value for tasks on that SchedTune CGroup */ + int boost; + + /* Performance Boost (B) region threshold params */ + int perf_boost_idx; + + /* Performance Constraint (C) region threshold params */ + int perf_constrain_idx; +}; + +static inline struct schedtune *css_st(struct cgroup_subsys_state *css) +{ + return css ? container_of(css, struct schedtune, css) : NULL; +} + +static inline struct schedtune *cgroup_st(struct cgroup *cgrp) { + return container_of(cgroup_subsys_state(cgrp, schedtune_subsys_id), struct schedtune, css); +} + +static inline struct schedtune *task_schedtune(struct task_struct *tsk) +{ + return cgroup_st(task_cgroup(tsk, schedtune_subsys_id)); +} + +#if 0 +static inline struct schedtune *parent_st(struct schedtune *st) +{ + return css_st(st->css.parent); +} +#endif + +/* + * SchedTune root control group + * The root control group is used to defined a system-wide boosting tuning, + * which is applied to all tasks in the system. + * Task specific boosting tuning could be specified by creating and + * configuring a child control group under the root one. + * By default, system-wide boosting is disabled, i.e. no boosting is applied + * to all tasks not into a child control group. + */ +static struct schedtune +root_schedtune = { + .boost = 0, + .perf_boost_idx = 0, + .perf_constrain_idx = 0, +}; + +int +schedtune_accept_deltas(int nrg_delta, int cap_delta, struct task_struct *task) { + struct schedtune *ct; + int perf_boost_idx; + int perf_constrain_idx; + + /* Optimal (O) region */ + if (nrg_delta < 0 && cap_delta > 0) { + // TJK: fix tracing: trace_sched_tune_filter(0, 0, 1, 0); + return INT_MAX; + } + + /* Suboptimal (S) region */ + if (nrg_delta > 0 && cap_delta < 0) { + // TJK: fix tracing trace_sched_tune_filter(0, 0, -1, 5); + return -INT_MAX; + } + + /* Get task specific perf Boost/Constraints indexes */ + rcu_read_lock(); + ct = task_schedtune(task); + perf_boost_idx = ct->perf_boost_idx; + perf_constrain_idx = ct->perf_constrain_idx; + rcu_read_unlock(); + + return __schedtune_accept_deltas(nrg_delta, cap_delta, + perf_boost_idx, perf_constrain_idx); + +} + +/* + * Maximum number of boost groups to support + * When per-task boosting is used we still allows only limited number of + * boost groups for two main reasons: + * 1. on a real system we usually have only few classes of workloads which + * make sense to boost with different values (e.g. backgroud vs foreground + * tasks, interactive vs low-priority tasks) + * 2. a limited number allows for a simpler and more memory/time efficient + * implementation especially for the computation of the per-CPU boost + * value + */ +#define BOOSTGROUPS_COUNT 16 + +/* Array of configured boostgroups */ +static struct schedtune *allocated_group[BOOSTGROUPS_COUNT] = { + &root_schedtune, + NULL, +}; + +/* SchedTune boost groups + * Each CPU in the system could be affected by multiple boost groups, for + * example when a CPU has two RUNNABLE tasks beloging to two different boost + * groups and thus likely with different boost values. + * This data structure keep track of all the boost groups which could impact + * on a CPU. + * Since on each system we expect only a limited number of boost + * groups, here we use a simple array to keep track of the metrics required to + * compute the maximum per-CPU boosting value. + */ +struct boost_groups { + /* Maximum boost value for all RUNNABLE tasks on a CPU */ + bool idle; + unsigned boost_max; + struct { + /* The boost for tasks on that boost group */ + unsigned boost; + /* Count of RUNNABLE tasks on that boost group */ + unsigned tasks; + } group[BOOSTGROUPS_COUNT]; +}; + +/* Boost groups affecting each CPU in the system */ +DEFINE_PER_CPU(struct boost_groups, cpu_boost_groups); + +static void +schedtune_cpu_update(int cpu) +{ + struct boost_groups *bg; + unsigned boost_max; + int idx; + + bg = &per_cpu(cpu_boost_groups, cpu); + + /* The root boost group is always active */ + boost_max = bg->group[0].boost; + for (idx = 1; idx < BOOSTGROUPS_COUNT; ++idx) { + /* + * A boost group affects a CPU only if it has + * RUNNABLE tasks on that CPU + */ + if (bg->group[idx].tasks == 0) + continue; + + boost_max = max(boost_max, bg->group[idx].boost); + } + + bg->boost_max = boost_max; +} + +static int +schedtune_boostgroup_update(int idx, int boost) +{ + struct boost_groups *bg; + int cur_boost_max; + int old_boost; + int cpu; + + /* Update per CPU boost groups */ + for_each_possible_cpu(cpu) { + bg = &per_cpu(cpu_boost_groups, cpu); + + /* + * Keep track of current boost values to compute the per CPU + * maximum only when it has been affected by the new value of + * the updated boost group + */ + cur_boost_max = bg->boost_max; + old_boost = bg->group[idx].boost; + + /* Update the boost value of this boost group */ + bg->group[idx].boost = boost; + + /* Check if this update increase current max */ + if (boost > cur_boost_max && bg->group[idx].tasks) { + bg->boost_max = boost; + // TJK: fix tracing: trace_sched_tune_boostgroup_update(cpu, 1, bg->boost_max); + continue; + } + + /* Check if this update has decreased current max */ + if (cur_boost_max == old_boost && old_boost > boost) { + schedtune_cpu_update(cpu); + // TJK: fix tracing: trace_sched_tune_boostgroup_update(cpu, -1, bg->boost_max); + continue; + } + + //TJK: fix tracing: trace_sched_tune_boostgroup_update(cpu, 0, bg->boost_max); + } + + return 0; +} + +void +schedtune_idle(int cpu) +{ + int idx; + struct boost_groups *bg; + + bg = &per_cpu(cpu_boost_groups, cpu); + if (!bg->idle) { + bg->idle = true; + for (idx = 1; idx < BOOSTGROUPS_COUNT; ++idx) { + bg->group[idx].tasks = 0; + } + } +} + +static inline void +schedtune_tasks_update(struct task_struct *p, int cpu, int idx, int task_count) +{ + struct boost_groups *bg; + int tasks; + + bg = &per_cpu(cpu_boost_groups, cpu); + bg->idle = false; + + /* Update boosted tasks count while avoiding to make it negative */ + if (task_count < 0 && bg->group[idx].tasks <= -task_count) + bg->group[idx].tasks = 0; + else + bg->group[idx].tasks += task_count; + + /* Boost group activation or deactivation on that RQ */ + tasks = bg->group[idx].tasks; + if (tasks == 1 || tasks == 0) + schedtune_cpu_update(cpu); + +#if 0 + // TJK: fix tracing + trace_sched_tune_tasks_update(p, cpu, tasks, idx, + bg->group[idx].boost, bg->boost_max); +#endif + +} + +/* + * NOTE: This function must be called while holding the lock on the CPU RQ + */ +void schedtune_enqueue_task(struct task_struct *p, int cpu) +{ + struct schedtune *st; + int idx; + + /* + * When a task is marked PF_EXITING by do_exit() it's going to be + * dequeued and enqueued multiple times in the exit path. + * Thus we avoid any further update, since we do not want to change + * CPU boosting while the task is exiting. + */ + if (p->flags & PF_EXITING) + return; + + /* Get task boost group */ + rcu_read_lock(); + st = task_schedtune(p); + idx = st->idx; + rcu_read_unlock(); + + schedtune_tasks_update(p, cpu, idx, 1); +} + +/* + * NOTE: This function must be called while holding the lock on the CPU RQ + */ +void schedtune_dequeue_task(struct task_struct *p, int cpu) +{ + struct schedtune *st; + int idx; + + /* + * When a task is marked PF_EXITING by do_exit() it's going to be + * dequeued and enqueued multiple times in the exit path. + * Thus we avoid any further update, since we do not want to change + * CPU boosting while the task is exiting. + * The last dequeue will be done by cgroup exit() callback. + */ + if (p->flags & PF_EXITING) + return; + + /* Get task boost group */ + rcu_read_lock(); + st = task_schedtune(p); + idx = st->idx; + rcu_read_unlock(); + + schedtune_tasks_update(p, cpu, idx, -1); +} + +int schedtune_taskgroup_boost(struct task_struct *p) +{ + struct schedtune *ct; + int task_boost; + + rcu_read_lock(); + ct = task_schedtune(p); + task_boost = ct->boost; + rcu_read_unlock(); + + return task_boost; +} + +int schedtune_cpu_boost(int cpu) +{ + struct boost_groups *bg; + bg = &per_cpu(cpu_boost_groups, cpu); + return bg->boost_max; +} + +static u64 +boost_read(struct cgroup *cgrp, struct cftype *cft) +{ + struct schedtune *st = cgroup_st(cgrp); + return st->boost; +} + +static int +boost_write(struct cgroup *cgrp, struct cftype *cft, + u64 boost) +{ + struct schedtune *st = cgroup_st(cgrp); + int err = 0; + + if (boost < 0 || boost > 100) { + err = -EINVAL; + goto out; + } + + st->boost = boost; + if (cgrp == (void *)&root_schedtune.css) + sysctl_sched_cfs_boost = boost; + + if (boost == 100) + boost = 99; + + /* Performance Boost (B) region threshold params */ + st->perf_boost_idx = boost; + st->perf_boost_idx /= 10; + + /* Performance Constraint (C) region threshold params */ + st->perf_constrain_idx = 100 - boost; + st->perf_constrain_idx /= 10; + + /* Update CPU boost */ + schedtune_boostgroup_update(st->idx, st->boost); +#if 0 + // TJK: fix tracing + trace_sched_tune_config(st->boost, + threshold_gains[st->perf_boost_idx].nrg_gain, + threshold_gains[st->perf_boost_idx].cap_gain, + threshold_gains[st->perf_constrain_idx].nrg_gain, + threshold_gains[st->perf_constrain_idx].cap_gain); +#endif +out: + return err; +} + +static struct cftype files[] = { + { + .name = "boost", + .read_u64 = boost_read, + .write_u64 = boost_write, + }, + { } /* terminate */ +}; + +static int +schedtune_boostgroup_init(struct schedtune *st) +{ + struct boost_groups *bg; + int cpu; + + /* Keep track of allocated boost group */ + allocated_group[st->idx] = st; + + /* Initialize the per CPU boost groups */ + for_each_possible_cpu(cpu) { + bg = &per_cpu(cpu_boost_groups, cpu); + bg->group[st->idx].boost = 0; + bg->group[st->idx].tasks = 0; + } + + return 0; +} + +static int +schedtune_init(void) +{ + struct boost_groups *bg; + int cpu; + + /* Initialize the per CPU boost groups */ + for_each_possible_cpu(cpu) { + bg = &per_cpu(cpu_boost_groups, cpu); + memset(bg, 0 , sizeof(struct boost_groups)); + } + + pr_info(" schedtune configured to support %d boost groups\n", + BOOSTGROUPS_COUNT); + return 0; +} + +static struct cgroup_subsys_state * +schedtune_css_alloc(struct cgroup *cgrp) +{ + struct schedtune *st; + int idx; + + if (!cgrp->parent) { + schedtune_init(); + return &root_schedtune.css; + } + + /* Allows only single level hierachies */ + if (cgrp->parent != (void *)&root_schedtune.css) { + pr_err("Nested SchedTune boosting groups not allowed\n"); + return ERR_PTR(-ENOMEM); + } + + /* Allows only a limited number of boosting groups */ + for (idx = 1; idx < BOOSTGROUPS_COUNT; ++idx) + if (allocated_group[idx] == NULL) + break; + if (idx == BOOSTGROUPS_COUNT) { + pr_err("Trying to create more than %d SchedTune boosting groups\n", + BOOSTGROUPS_COUNT); + return ERR_PTR(-ENOSPC); + } + + st = kzalloc(sizeof(*st), GFP_KERNEL); + if (!st) + goto out; + + /* Initialize per CPUs boost group support */ + st->idx = idx; + if (schedtune_boostgroup_init(st)) + goto release; + + return &st->css; + +release: + kfree(st); +out: + return ERR_PTR(-ENOMEM); +} + +static void +schedtune_boostgroup_release(struct schedtune *st) +{ + /* Reset this group boost */ + schedtune_boostgroup_update(st->idx, 0); + + /* Keep track of allocated boost group */ + allocated_group[st->idx] = NULL; +} + +static void +schedtune_css_free(struct cgroup *cgrp) +{ + struct schedtune *st = cgroup_st(cgrp); + schedtune_boostgroup_release(st); + kfree(st); +} + +static void +schedtune_exit(struct cgroup *cgrp, + struct cgroup *old_css, + struct task_struct *tsk) +{ + struct schedtune *old_st = cgroup_st(old_css); + int cpu = task_cpu(tsk); + + schedtune_tasks_update(tsk, cpu, old_st->idx, -1); +} + +static int +schedtune_allow_attach(struct cgroup *cgrp, struct cgroup_taskset *tset) { + // TJK: too permissive, but ok for now + return 0; +} + +struct cgroup_subsys schedtune_subsys = { + .name = "schedtune", + .css_alloc = schedtune_css_alloc, + .css_free = schedtune_css_free, + .exit = schedtune_exit, + .allow_attach = schedtune_allow_attach, + .base_cftypes = files, + .subsys_id = schedtune_subsys_id, + .early_init = 1, +}; + +#else /* CONFIG_CGROUP_SCHEDTUNE */ + +int +schedtune_accept_deltas(int nrg_delta, int cap_delta) { + + /* Optimal (O) region */ + if (nrg_delta < 0 && cap_delta > 0) { + trace_printk("schedtune_filter: region=O ngain=0 pgain=0 nrg_payoff=-1"); + return INT_MAX; + } + + /* Suboptimal (S) region */ + if (nrg_delta > 0 && cap_delta < 0) { + trace_printk("schedtune_filter: region=S ngain=0 pgain=0 nrg_payoff=1"); + return -INT_MAX; + } + + return __schedtune_accept_deltas(nrg_delta, cap_delta, + perf_boost_idx, perf_constrain_idx); + +} + +#endif /* CONFIG_CGROUP_SCHEDTUNE */ + +int +sysctl_sched_cfs_boost_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret || !write) + return ret; + + /* Performance Boost (B) region threshold params */ + perf_boost_idx = sysctl_sched_cfs_boost; + perf_boost_idx /= 10; + + /* Performance Constraint (C) region threshold params */ + perf_constrain_idx = 100 - sysctl_sched_cfs_boost; + perf_constrain_idx /= 10; + + return 0; +} + diff --git a/kernel/sched/tune.h b/kernel/sched/tune.h new file mode 100644 index 0000000000000..2332d30cf383a --- /dev/null +++ b/kernel/sched/tune.h @@ -0,0 +1,35 @@ + +#ifdef CONFIG_SCHED_TUNE + +extern int schedtune_normalize_energy(int energy); + +#ifdef CONFIG_CGROUP_SCHEDTUNE + +extern int schedtune_taskgroup_boost(struct task_struct *tsk); +extern int schedtune_cpu_boost(int cpu); +extern void schedtune_idle(int cpu); + +extern void schedtune_enqueue_task(struct task_struct *p, int cpu); +extern void schedtune_dequeue_task(struct task_struct *p, int cpu); + +extern int schedtune_accept_deltas(int nrg_delta, int cap_delta, + struct task_struct *task); + +#else /* CONFIG_CGROUP_SCHEDTUNE */ + +extern int schedtune_accept_deltas(int nrg_delta, int cap_delta); + +#define schedtune_enqueue_task(task, cpu) while(0){} +#define schedtune_dequeue_task(task, cpu) while(0){} + +#endif /* CONFIG_CGROUP_SCHEDTUNE */ + +#else /* CONFIG_SCHED_TUNE */ + +#define schedtune_normalize_energy(energy) energy +#define schedtune_accept_deltas(nrg_delta, cap_delta) nrg_delta + +#define schedtune_enqueue_task(task, cpu) while(0){} +#define schedtune_dequeue_task(task, cpu) while(0){} + +#endif /* CONFIG_SCHED_TUNE */ From 6f0ec80044d79ac7c92c0616be0ea679ad30aeb5 Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Tue, 15 Dec 2015 16:36:22 -0800 Subject: [PATCH 12/86] Add support for making sched-dvfs the default cpufreq governor Change-Id: Ib0ea99b1778e6fcd3a7e90c5a70886654b66bb8a --- include/linux/cpufreq.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index d3c1819dc0d48..b7850b9cd62e0 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -432,6 +432,9 @@ extern struct cpufreq_governor cpufreq_gov_conservative; #elif defined(CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE) extern struct cpufreq_governor cpufreq_gov_interactive; #define CPUFREQ_DEFAULT_GOVERNOR (&cpufreq_gov_interactive) +#elif defined(CONFIG_CPU_FREQ_DEFAULT_GOV_SCHED) +extern struct cpufreq_governor cpufreq_gov_sched; +#define CPUFREQ_DEFAULT_GOVERNOR (&cpufreq_gov_sched) #endif /********************************************************************* From 3209e0638618e883fd2b77fbf3111d9b1bfc2913 Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Wed, 16 Dec 2015 13:49:37 -0800 Subject: [PATCH 13/86] Notify scheduler of cpuidle state changes Change-Id: If48c8c5f2347113d90fa40e7a4a335e1fb57ce69 --- drivers/cpuidle/lpm-levels.c | 4 ++-- include/linux/sched.h | 5 ++++- kernel/sched/core.c | 13 ++++++------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/drivers/cpuidle/lpm-levels.c b/drivers/cpuidle/lpm-levels.c index ad0d407f92bac..e93b587102aba 100644 --- a/drivers/cpuidle/lpm-levels.c +++ b/drivers/cpuidle/lpm-levels.c @@ -696,7 +696,7 @@ static int lpm_cpuidle_enter(struct cpuidle_device *dev, } pwr_params = &cluster->cpu->levels[idx].pwr; - sched_set_cpu_cstate(smp_processor_id(), idx + 1, + sched_set_cpu_cstate(smp_processor_id(), idx + 1, &drv->states[idx+1], pwr_params->energy_overhead, pwr_params->latency_us); cpu_prepare(cluster, idx, true); @@ -710,7 +710,7 @@ static int lpm_cpuidle_enter(struct cpuidle_device *dev, cluster_unprepare(cluster, cpumask, idx, true); cpu_unprepare(cluster, idx, true); - sched_set_cpu_cstate(smp_processor_id(), 0, 0, 0); + sched_set_cpu_cstate(smp_processor_id(), 0, &drv->states[0], 0, 0); time = ktime_to_ns(ktime_get()) - time; do_div(time, 1000); diff --git a/include/linux/sched.h b/include/linux/sched.h index fe5132d5a04af..842d9c4c3d5d8 100755 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2219,12 +2219,14 @@ static inline void tsk_restore_flags(struct task_struct *task, } #ifdef CONFIG_SMP +struct cpuidle_state; extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask); extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask); extern void sched_set_cpu_cstate(int cpu, int cstate, + struct cpuidle_state *cpuidle, int wakeup_energy, int wakeup_latency); extern void sched_set_cluster_dstate(const cpumask_t *cluster_cpus, int dstate, int wakeup_energy, int wakeup_latency); @@ -2241,7 +2243,8 @@ static inline int set_cpus_allowed_ptr(struct task_struct *p, return 0; } static inline void -sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency) +sched_set_cpu_cstate(int cpu, int cstate, struct cpuidle *cpuidle, + int wakeup_energy, int wakeup_latency) { } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0c578d9e2c473..1d73c36655d6a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -8812,18 +8812,18 @@ EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ * @wakeup_latency = latency to wakeup from cstate * */ + void -sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency) +sched_set_cpu_cstate(int cpu, int cstate, struct cpuidle_state *cpuidle, + int wakeup_energy, int wakeup_latency) { // TJK: Need to integrate QC cpuidle with EAS (doesn't use standard // cpuidle framework) -#if 0 struct rq *rq = cpu_rq(cpu); - rq->cstate = cstate; /* C1, C2 etc */ - rq->wakeup_energy = wakeup_energy; - rq->wakeup_latency = wakeup_latency; -#else + idle_set_state_idx(rq, cstate); + idle_set_state(rq, cpuidle); + if ((cstate == 0 && wakeup_energy == 0 && wakeup_latency == 0) || (cstate == 1 && cpu < 4 && wakeup_energy == 5800 && wakeup_latency == 50) || (cstate == 1 && cpu >= 4 && wakeup_energy == 24120 && wakeup_latency == 60) || @@ -8833,6 +8833,5 @@ sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency) } else { pr_info("TJK: CPU%d ->cstate=%d nrg=%d latency=%d\n", cpu, cstate, wakeup_energy, wakeup_latency); } -#endif } From f3c367046c3a7421779bb485f7b34920b0f2ca0e Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Wed, 16 Dec 2015 17:32:38 -0800 Subject: [PATCH 14/86] Remove incorrect test for root node in sched-tune Change-Id: I04de63d31e12249f83655c29b91f9adb11d2ae47 --- kernel/sched/tune.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/tune.c b/kernel/sched/tune.c index 779bb0f658535..dff4978e2afd2 100644 --- a/kernel/sched/tune.c +++ b/kernel/sched/tune.c @@ -596,11 +596,12 @@ schedtune_css_alloc(struct cgroup *cgrp) } /* Allows only single level hierachies */ +#if 0 if (cgrp->parent != (void *)&root_schedtune.css) { pr_err("Nested SchedTune boosting groups not allowed\n"); return ERR_PTR(-ENOMEM); } - +#endif /* Allows only a limited number of boosting groups */ for (idx = 1; idx < BOOSTGROUPS_COUNT; ++idx) if (allocated_group[idx] == NULL) From aa52c30517707f6271b2cbfa0387573c191cc92e Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Thu, 17 Dec 2015 10:11:30 -0800 Subject: [PATCH 15/86] Enable EAS ftrace events Change-Id: Ic1f0f90b84a71acdfcb4d40664f6571b7739b401 --- include/trace/events/sched.h | 343 +++++++++++++++++++++++++++++++++++ kernel/sched/core.c | 43 +---- kernel/sched/fair.c | 18 +- kernel/sched/tune.c | 26 +-- 4 files changed, 359 insertions(+), 71 deletions(-) diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index 493bdf554bb4c..0d0574f3a1241 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -948,6 +948,349 @@ TRACE_EVENT(sched_get_nr_running_avg, __entry->avg, __entry->big_avg, __entry->iowait_avg) ); +/* + * Tracepoint for accounting sched averages for tasks. + */ +TRACE_EVENT(sched_load_avg_task, + + TP_PROTO(struct task_struct *tsk, struct sched_avg *avg), + + TP_ARGS(tsk, avg), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( int, cpu ) + __field( unsigned long, load ) + __field( unsigned long, utilization ) + __field( unsigned int, runnable_avg_sum ) + __field( unsigned int, running_avg_sum ) + __field( unsigned int, avg_period ) + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->cpu = task_cpu(tsk); + __entry->load = avg->load_avg_contrib; + __entry->utilization = avg->utilization_avg_contrib; + __entry->runnable_avg_sum = avg->runnable_avg_sum; + __entry->running_avg_sum = avg->running_avg_sum; + __entry->avg_period = avg->avg_period; + ), + + TP_printk("comm=%s pid=%d cpu=%d load=%lu utilization=%lu runnable_avg_sum=%u" + " running_avg_sum=%u avg_period=%u", + __entry->comm, __entry->pid, __entry->cpu, + __entry->load, __entry->utilization, + (unsigned int)__entry->runnable_avg_sum, + (unsigned int)__entry->running_avg_sum, + (unsigned int)__entry->avg_period) +); + +/* + * Tracepoint for accounting sched group energy + */ +TRACE_EVENT(sched_energy_diff, + + TP_PROTO(struct task_struct *tsk, int scpu, int dcpu, int udelta, + int nrgb, int nrga, int nrgd, int capb, int capa, int capd, + int nrgn, int nrgp), + + TP_ARGS(tsk, scpu, dcpu, udelta, + nrgb, nrga, nrgd, capb, capa, capd, + nrgn, nrgp), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( int, scpu ) + __field( int, dcpu ) + __field( int, udelta ) + __field( int, nrgb ) + __field( int, nrga ) + __field( int, nrgd ) + __field( int, capb ) + __field( int, capa ) + __field( int, capd ) + __field( int, nrgn ) + __field( int, nrgp ) + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->scpu = scpu; + __entry->dcpu = dcpu; + __entry->udelta = udelta; + __entry->nrgb = nrgb; + __entry->nrga = nrga; + __entry->nrgd = nrgd; + __entry->capb = capb; + __entry->capa = capa; + __entry->capd = capd; + __entry->nrgn = nrgn; + __entry->nrgp = nrgp; + ), + + TP_printk("pid=%d comm=%s " + "src_cpu=%d dst_cpu=%d usage_delta=%d " + "nrg_before=%d nrg_after=%d nrg_diff=%d " + "cap_before=%d cap_after=%d cap_delta=%d " + "nrg_delta=%d nrg_payoff=%d", + __entry->pid, __entry->comm, + __entry->scpu, __entry->dcpu, __entry->udelta, + __entry->nrgb, __entry->nrga, __entry->nrgd, + __entry->capb, __entry->capa, __entry->capd, + __entry->nrgn, __entry->nrgp) +); + + +/* + * Tracepoint for waking a polling cpu without an IPI. + */ +TRACE_EVENT(sched_wake_idle_without_ipi, + + TP_PROTO(int cpu), + + TP_ARGS(cpu), + + TP_STRUCT__entry( + __field( int, cpu ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + ), + + TP_printk("cpu=%d", __entry->cpu) +); + +TRACE_EVENT(sched_contrib_scale_f, + + TP_PROTO(int cpu, unsigned long freq_scale_factor, + unsigned long cpu_scale_factor), + + TP_ARGS(cpu, freq_scale_factor, cpu_scale_factor), + + TP_STRUCT__entry( + __field(int, cpu) + __field(unsigned long, freq_scale_factor) + __field(unsigned long, cpu_scale_factor) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->freq_scale_factor = freq_scale_factor; + __entry->cpu_scale_factor = cpu_scale_factor; + ), + + TP_printk("cpu=%d freq_scale_factor=%lu cpu_scale_factor=%lu", + __entry->cpu, __entry->freq_scale_factor, + __entry->cpu_scale_factor) +); + +TRACE_EVENT(sched_load_avg_cpu, + + TP_PROTO(int cpu, struct cfs_rq *cfs_rq), + + TP_ARGS(cpu, cfs_rq), + + TP_STRUCT__entry( + __field( int, cpu ) + __field( unsigned long, load ) + __field( unsigned long, utilization ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->load = cfs_rq->runnable_load_avg; + __entry->utilization = cfs_rq->utilization_load_avg; + ), + + TP_printk("cpu=%d load=%lu utilization=%lu", + __entry->cpu, __entry->load, __entry->utilization) +); + +/* + * Tracepoint for accounting task boosted utilization + */ +TRACE_EVENT(sched_boost_task, + + TP_PROTO(struct task_struct *tsk, unsigned long utilization, unsigned long margin), + + TP_ARGS(tsk, utilization, margin), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( unsigned long, utilization ) + __field( unsigned long, margin ) + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->utilization = utilization; + __entry->margin = margin; + ), + + TP_printk("comm=%s pid=%d utilization=%lu margin=%lu", + __entry->comm, __entry->pid, + __entry->utilization, + __entry->margin) +); + +/* + * Tracepoint for accounting CPU boosted utilization + */ +TRACE_EVENT(sched_boost_cpu, + + TP_PROTO(int cpu, unsigned long usage, unsigned long margin), + + TP_ARGS(cpu, usage, margin), + + TP_STRUCT__entry( + __field( int, cpu ) + __field( unsigned long, usage ) + __field( unsigned long, margin ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->usage = usage; + __entry->margin = margin; + ), + + TP_printk("cpu=%d usage=%lu margin=%lu", + __entry->cpu, + __entry->usage, + __entry->margin) +); + +/* + * Tracepoint for schedtune_tasks_update + */ +TRACE_EVENT(sched_tune_filter, + + TP_PROTO(int nrg_delta, int cap_delta, int nrg_payoff, int region), + + TP_ARGS(nrg_delta, cap_delta, nrg_payoff, region), + + TP_STRUCT__entry( + __field( int, nrg_delta ) + __field( int, cap_delta ) + __field( int, nrg_payoff ) + __field( int, region ) + ), + + TP_fast_assign( + __entry->nrg_delta = nrg_delta; + __entry->cap_delta = cap_delta; + __entry->nrg_payoff = nrg_payoff; + __entry->region = region; + ), + + TP_printk("nrg_delta=%d cap_delta=%d nrg_payoff=%d region=%d", + __entry->nrg_delta, __entry->cap_delta, + __entry->nrg_payoff, __entry->region) +); + +/* + * Tracepoint for schedtune_boostgroup_update + */ +TRACE_EVENT(sched_tune_boostgroup_update, + + TP_PROTO(int cpu, int variation, int max_boost), + + TP_ARGS(cpu, variation, max_boost), + + TP_STRUCT__entry( + __field( int, cpu ) + __field( int, variation ) + __field( int, max_boost ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->variation = variation; + __entry->max_boost = max_boost; + ), + + TP_printk("cpu=%d variation=%d max_boost=%d", + __entry->cpu, __entry->variation, __entry->max_boost) +); + +/* + * Tracepoint for schedtune_tasks_update + */ +TRACE_EVENT(sched_tune_tasks_update, + + TP_PROTO(struct task_struct *tsk, int cpu, int tasks, int idx, + unsigned int boost, unsigned int max_boost), + + TP_ARGS(tsk, cpu, tasks, idx, boost, max_boost), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( int, cpu ) + __field( int, tasks ) + __field( int, idx ) + __field( unsigned int, boost ) + __field( unsigned int, max_boost ) + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->cpu = cpu; + __entry->tasks = tasks; + __entry->idx = idx; + __entry->boost = boost; + __entry->max_boost = max_boost; + ), + + TP_printk("pid=%d comm=%s " + "cpu=%d tasks=%d idx=%d boost=%u max_boost=%u", + __entry->pid, __entry->comm, + __entry->cpu, __entry->tasks, __entry->idx, + __entry->boost, __entry->max_boost) +); + +/* + * Tracepoint for sched_tune_config settings + */ +TRACE_EVENT(sched_tune_config, + + TP_PROTO(int boost, int pb_nrg_gain, int pb_cap_gain, int pc_nrg_gain, int pc_cap_gain), + + TP_ARGS(boost, pb_nrg_gain, pb_cap_gain, pc_nrg_gain, pc_cap_gain), + + TP_STRUCT__entry( + __field( int, boost ) + __field( int, pb_nrg_gain ) + __field( int, pb_cap_gain ) + __field( int, pc_nrg_gain ) + __field( int, pc_cap_gain ) + ), + + TP_fast_assign( + __entry->boost = boost; + __entry->pb_nrg_gain = pb_nrg_gain; + __entry->pb_cap_gain = pb_cap_gain; + __entry->pc_nrg_gain = pc_nrg_gain; + __entry->pc_cap_gain = pc_cap_gain; + ), + + TP_printk("boost=%d " + "pb_nrg_gain=%d pb_cap_gain=%d " + "pc_nrg_gain=%d pc_cap_gain=%d", + __entry->boost, + __entry->pb_nrg_gain, __entry->pb_cap_gain, + __entry->pc_nrg_gain, __entry->pc_cap_gain) +); + #endif /* _TRACE_SCHED_H */ /* This part must be outside protection */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 1d73c36655d6a..1f341763ea909 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -626,11 +626,8 @@ void resched_curr(struct rq *rq) if (set_nr_and_not_polling(curr)) smp_send_reschedule(cpu); - // TJK fix tracing -#if 0 else trace_sched_wake_idle_without_ipi(cpu); -#endif } void resched_cpu(int cpu) @@ -692,11 +689,8 @@ static void wake_up_idle_cpu(int cpu) if (set_nr_and_not_polling(rq->idle)) smp_send_reschedule(cpu); -#if 0 - // TJK fix tracing else trace_sched_wake_idle_without_ipi(cpu); -#endif } static bool wake_up_full_nohz_cpu(int cpu) @@ -1083,7 +1077,6 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) #endif #endif - // TJK: convert to new trace args trace_sched_migrate_task(p, new_cpu, 0); if (task_cpu(p) != new_cpu) { @@ -1653,11 +1646,8 @@ static void ttwu_queue_remote(struct task_struct *p, int cpu) if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) { if (!set_nr_if_polling(rq->idle)) smp_send_reschedule(cpu); -#if 0 - // TJK: fix tracing else trace_sched_wake_idle_without_ipi(cpu); -#endif } } @@ -1672,7 +1662,7 @@ void wake_up_if_idle(int cpu) goto out; if (set_nr_if_polling(rq->idle)) { - // TJK: trace_sched_wake_idle_without_ipi(cpu); + trace_sched_wake_idle_without_ipi(cpu); } else { raw_spin_lock_irqsave(&rq->lock, flags); if (is_idle_task(rq->curr)) @@ -2703,37 +2693,6 @@ void preempt_count_add(int val) } } EXPORT_SYMBOL(preempt_count_add); -#if 0 -// TJK -NOKPROBE_SYMBOL(preempt_count_add); - -void preempt_count_sub(int val) -{ -#ifdef CONFIG_DEBUG_PREEMPT - /* - * Underflow? - */ - if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) - return; - /* - * Is the spinlock portion underflowing? - */ - if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && - !(preempt_count() & PREEMPT_MASK))) - return; -#endif - - if (preempt_count() == val) - trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); -#if defined(CONFIG_PREEMPT_MONITOR) && defined(CONFIG_MTPROF) - if (unlikely(__raw_get_cpu_var(mtsched_mon_enabled) & 0x1)) - MT_trace_preempt_on(); -#endif - __preempt_count_sub(val); -} -EXPORT_SYMBOL(preempt_count_sub); -NOKPROBE_SYMBOL(preempt_count_sub); -#endif #endif diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index efc7128798854..77b5496baef29 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2294,8 +2294,7 @@ static __always_inline int __update_entity_runnable_avg(u64 now, int cpu, unsigned long scale_freq = arch_scale_freq_capacity(NULL, cpu); unsigned long scale_cpu = arch_scale_cpu_capacity(NULL, cpu); - //TJK fix tracing: trace_sched_contrib_scale_f(cpu, scale_freq, scale_cpu); - + trace_sched_contrib_scale_f(cpu, scale_freq, scale_cpu); delta = now - sa->last_runnable_update; /* @@ -2595,11 +2594,9 @@ static inline void update_entity_load_avg(struct sched_entity *se, contrib_delta = __update_entity_load_avg_contrib(se); utilization_delta = __update_entity_utilization_avg_contrib(se); -#if 0 - // TJK: fix tracing if (entity_is_task(se)) trace_sched_load_avg_task(task_of(se), &se->avg); -#endif + if (!update_cfs_rq) return; @@ -2612,7 +2609,7 @@ static inline void update_entity_load_avg(struct sched_entity *se, -utilization_delta); } - // TJK: fix tracing: trace_sched_load_avg_cpu(cpu, cfs_rq); + trace_sched_load_avg_cpu(cpu, cfs_rq); } /* @@ -4814,14 +4811,13 @@ static int energy_diff(struct energy_env *eenv) eenv->energy_payoff = 0; result = energy_diff_evaluate(eenv); -#if 0 - // TJK: fix tracint + trace_sched_energy_diff(eenv->task, eenv->src_cpu, eenv->dst_cpu, eenv->usage_delta, eenv->nrg.before, eenv->nrg.after, eenv->nrg.diff, eenv->cap.before, eenv->cap.after, eenv->cap.delta, eenv->nrg.delta, eenv->energy_payoff); -#endif + return result; } @@ -4998,7 +4994,7 @@ boosted_task_utilization(struct task_struct *task) if (!task_rq(task)->rd->overutilized) margin = schedtune_task_margin(task); - //TJK: fix tracing: trace_sched_boost_task(task, utilization, margin); + trace_sched_boost_task(task, utilization, margin); utilization += margin; @@ -5088,7 +5084,7 @@ get_boosted_cpu_usage(int cpu) usage = get_cpu_usage(cpu); margin = schedtune_cpu_margin(cpu, usage); - // TJK: fix tracing trace_sched_boost_cpu(cpu, usage, margin); + trace_sched_boost_cpu(cpu, usage, margin); usage += margin; return usage; diff --git a/kernel/sched/tune.c b/kernel/sched/tune.c index dff4978e2afd2..79de07ed75759 100644 --- a/kernel/sched/tune.c +++ b/kernel/sched/tune.c @@ -137,13 +137,12 @@ __schedtune_accept_deltas(int nrg_delta, int cap_delta, */ energy_payoff = cap_delta * threshold_gains[perf_boost_idx].nrg_gain; energy_payoff -= nrg_delta * threshold_gains[perf_boost_idx].cap_gain; -#if 0 - // TJK: fix tracing + trace_sched_tune_filter( threshold_gains[perf_boost_idx].nrg_gain, threshold_gains[perf_boost_idx].cap_gain, energy_payoff, 8); -#endif + return energy_payoff; } @@ -158,13 +157,11 @@ __schedtune_accept_deltas(int nrg_delta, int cap_delta, energy_payoff = nrg_delta * threshold_gains[perf_constrain_idx].cap_gain; energy_payoff -= cap_delta * threshold_gains[perf_constrain_idx].nrg_gain; -#if 0 - // TJK: fix tracing trace_sched_tune_filter( threshold_gains[perf_constrain_idx].nrg_gain, threshold_gains[perf_constrain_idx].cap_gain, energy_payoff, 6); -#endif + return energy_payoff; } @@ -241,13 +238,13 @@ schedtune_accept_deltas(int nrg_delta, int cap_delta, struct task_struct *task) /* Optimal (O) region */ if (nrg_delta < 0 && cap_delta > 0) { - // TJK: fix tracing: trace_sched_tune_filter(0, 0, 1, 0); + trace_sched_tune_filter(0, 0, 1, 0); return INT_MAX; } /* Suboptimal (S) region */ if (nrg_delta > 0 && cap_delta < 0) { - // TJK: fix tracing trace_sched_tune_filter(0, 0, -1, 5); + trace_sched_tune_filter(0, 0, -1, 5); return -INT_MAX; } @@ -358,18 +355,18 @@ schedtune_boostgroup_update(int idx, int boost) /* Check if this update increase current max */ if (boost > cur_boost_max && bg->group[idx].tasks) { bg->boost_max = boost; - // TJK: fix tracing: trace_sched_tune_boostgroup_update(cpu, 1, bg->boost_max); + trace_sched_tune_boostgroup_update(cpu, 1, bg->boost_max); continue; } /* Check if this update has decreased current max */ if (cur_boost_max == old_boost && old_boost > boost) { schedtune_cpu_update(cpu); - // TJK: fix tracing: trace_sched_tune_boostgroup_update(cpu, -1, bg->boost_max); + trace_sched_tune_boostgroup_update(cpu, -1, bg->boost_max); continue; } - //TJK: fix tracing: trace_sched_tune_boostgroup_update(cpu, 0, bg->boost_max); + trace_sched_tune_boostgroup_update(cpu, 0, bg->boost_max); } return 0; @@ -410,12 +407,8 @@ schedtune_tasks_update(struct task_struct *p, int cpu, int idx, int task_count) if (tasks == 1 || tasks == 0) schedtune_cpu_update(cpu); -#if 0 - // TJK: fix tracing trace_sched_tune_tasks_update(p, cpu, tasks, idx, bg->group[idx].boost, bg->boost_max); -#endif - } /* @@ -527,14 +520,11 @@ boost_write(struct cgroup *cgrp, struct cftype *cft, /* Update CPU boost */ schedtune_boostgroup_update(st->idx, st->boost); -#if 0 - // TJK: fix tracing trace_sched_tune_config(st->boost, threshold_gains[st->perf_boost_idx].nrg_gain, threshold_gains[st->perf_boost_idx].cap_gain, threshold_gains[st->perf_constrain_idx].nrg_gain, threshold_gains[st->perf_constrain_idx].cap_gain); -#endif out: return err; } From 76b57b1b4e7b8a624611c17152f7a40f6020eefe Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Mon, 21 Dec 2015 17:02:59 -0800 Subject: [PATCH 16/86] EAS fixes to get energy model to work correctly - Added a tunable for the EAS margin: sched_capacity_margin - Better support for multi-cluster in sched-dvfs - Make sure "real" version of arch_scale_cpu_capacity is used Change-Id: I28088d4f48741fc8be4ca928e3a9be9a590a0700 --- arch/arm64/kernel/topology.c | 34 ++++++++++++++++++++++++--- include/linux/sched/sysctl.h | 2 +- include/linux/topology.h | 5 ++++ include/trace/events/sched.h | 41 ++++++++++++++++++++++++++++++++ kernel/sched/cpufreq_sched.c | 10 ++++---- kernel/sched/fair.c | 45 ++++++++++++++++++++---------------- kernel/sched/sched.h | 2 ++ kernel/sysctl.c | 4 ++-- 8 files changed, 113 insertions(+), 30 deletions(-) diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 245825c0fc2b1..0de1aa7b28cb2 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -281,8 +281,9 @@ unsigned long arch_scale_freq_capacity(struct sched_domain *sd, int cpu) { unsigned long curr = atomic_long_read(&per_cpu(cpu_freq_capacity, cpu)); - if (!curr) - return SCHED_CAPACITY_SCALE; + if (!curr) { + return SCHED_CAPACITY_SCALE; + } return curr; } @@ -513,11 +514,36 @@ static void __init reset_cpu_topology(void) static void __init reset_cpu_capacity(void) { unsigned int cpu; - for_each_possible_cpu(cpu) set_capacity_scale(cpu, SCHED_CAPACITY_SCALE); } +static inline const struct sched_group_energy *cpu_cluster_energy(int cpu) +{ + struct sched_group_energy *sge = sge_array[cpu][SD_LEVEL1]; + + if (!sge) { + pr_warn("Invalid sched_group_energy for Cluster%d\n", cpu); + return NULL; + } + + return sge; +} + +static inline int cpu_corepower_flags(void) +{ + return SD_SHARE_PKG_RESOURCES | SD_SHARE_POWERDOMAIN | \ + SD_SHARE_CAP_STATES; +} + +static struct sched_domain_topology_level arm64_topology[] = { +#ifdef CONFIG_SCHED_MC + { cpu_coregroup_mask, cpu_corepower_flags, cpu_core_energy, SD_INIT_NAME(MC) }, +#endif + { cpu_cpu_mask, 0, cpu_cluster_energy, SD_INIT_NAME(DIE) }, + { NULL, }, +}; + void __init init_cpu_topology(void) { reset_cpu_topology(); @@ -528,6 +554,8 @@ void __init init_cpu_topology(void) */ if (parse_dt_topology()) reset_cpu_topology(); + else + set_sched_topology(arm64_topology); reset_cpu_capacity(); parse_dt_cpu_capacity(); diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index e99174dd0afa0..b8648441b7970 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -40,7 +40,7 @@ extern unsigned int sysctl_sched_min_granularity; extern unsigned int sysctl_sched_wakeup_granularity; extern unsigned int sysctl_sched_child_runs_first; extern unsigned int sysctl_sched_wake_to_idle; -extern unsigned int sysctl_sched_wakeup_load_threshold; +extern unsigned int sysctl_sched_capacity_margin; extern unsigned int sysctl_sched_window_stats_policy; extern unsigned int sysctl_sched_account_wait_time; extern unsigned int sysctl_sched_ravg_hist_size; diff --git a/include/linux/topology.h b/include/linux/topology.h index 9138f31200a4c..8f1c8c870c9f4 100644 --- a/include/linux/topology.h +++ b/include/linux/topology.h @@ -289,4 +289,9 @@ static inline int cpu_to_mem(int cpu) #define topology_core_cpumask(cpu) cpumask_of(cpu) #endif +static inline const struct cpumask *cpu_cpu_mask(int cpu) +{ + return cpumask_of_node(cpu_to_node(cpu)); +} + #endif /* _LINUX_TOPOLOGY_H */ diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index 0d0574f3a1241..3162fc2ede02c 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -1141,6 +1141,47 @@ TRACE_EVENT(sched_boost_task, __entry->margin) ); +TRACE_EVENT(sched_task_fits, + + TP_PROTO(struct task_struct *tsk, int cpu, int orig_usage, int usage, unsigned long task_util, unsigned long capacity, unsigned long margin), + + TP_ARGS(tsk, cpu, orig_usage, usage, task_util, capacity, margin), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( int, cpu ) + __field( int, orig_usage ) + __field( int, usage ) + __field( unsigned long, task_util ) + __field( unsigned long, capacity ) + __field( unsigned long, margin ) + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->cpu = cpu; + __entry->orig_usage = orig_usage; + __entry->usage = usage; + __entry->capacity = capacity; + __entry->task_util = task_util; + __entry->margin = margin; + ), + + TP_printk("comm=%s pid=%d cpu=%d origu=%d tasku=%ld u=%d boost=%d capacity=%lu margin=%lu result=%d", + __entry->comm, __entry->pid, + __entry->cpu, + __entry->orig_usage, + __entry->task_util, + __entry->usage, + __entry->usage-__entry->orig_usage, + __entry->capacity, + __entry->margin, + (__entry->capacity * 1024) > (__entry->usage * __entry->margin) + ) +); + /* * Tracepoint for accounting CPU boosted utilization */ diff --git a/kernel/sched/cpufreq_sched.c b/kernel/sched/cpufreq_sched.c index 5f0b3fe15db67..c067462dc3402 100644 --- a/kernel/sched/cpufreq_sched.c +++ b/kernel/sched/cpufreq_sched.c @@ -221,14 +221,16 @@ void cpufreq_sched_reset_cap(int cpu) static inline void set_sched_energy_freq(void) { - if (!sched_energy_freq()) - static_key_slow_inc(&__sched_energy_freq); + // TJK: must always bump the key so we can handle + // 1 cluster going online/offline + static_key_slow_inc(&__sched_energy_freq); } static inline void clear_sched_energy_freq(void) { - if (sched_energy_freq()) - static_key_slow_dec(&__sched_energy_freq); + // TJK: must always bump the key so we can handle + // 1 cluster going online/offline + static_key_slow_dec(&__sched_energy_freq); } static int cpufreq_sched_policy_start(struct cpufreq_policy *policy) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 77b5496baef29..a40447dc21c0a 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -39,7 +39,6 @@ #ifndef TJK_HMP // TJK: resolve undefined sysctl symbols unsigned int __read_mostly sysctl_sched_wake_to_idle = 0; -__read_mostly unsigned int sysctl_sched_wakeup_load_threshold = 110; #endif @@ -2294,6 +2293,9 @@ static __always_inline int __update_entity_runnable_avg(u64 now, int cpu, unsigned long scale_freq = arch_scale_freq_capacity(NULL, cpu); unsigned long scale_cpu = arch_scale_cpu_capacity(NULL, cpu); +#define TK_PERIOD_SHIFT 10 +#define TK_PERIOD (1<last_runnable_update; @@ -2310,14 +2312,14 @@ static __always_inline int __update_entity_runnable_avg(u64 now, int cpu, * Use 1024ns as the unit of measurement since it's a reasonable * approximation of 1us and fast to compute. */ - delta >>= 10; + delta >>= TK_PERIOD_SHIFT; if (!delta) return 0; sa->last_runnable_update = now; /* delta_w is the amount already accumulated against our next period */ - delta_w = sa->avg_period % 1024; - if (delta + delta_w >= 1024) { + delta_w = sa->avg_period % TK_PERIOD; + if (delta + delta_w >= TK_PERIOD) { /* period roll-over */ decayed = 1; @@ -2326,7 +2328,7 @@ static __always_inline int __update_entity_runnable_avg(u64 now, int cpu, * out how much from delta we need to complete the current * period and accrue it. */ - delta_w = 1024 - delta_w; + delta_w = TK_PERIOD - delta_w; scaled_delta_w = (delta_w * scale_freq) >> SCHED_CAPACITY_SHIFT; if (runnable) @@ -2342,8 +2344,8 @@ static __always_inline int __update_entity_runnable_avg(u64 now, int cpu, delta -= delta_w; /* Figure out how many additional periods this update spans */ - periods = delta / 1024; - delta %= 1024; + periods = delta / TK_PERIOD; + delta %= TK_PERIOD; sa->runnable_avg_sum = decay_load(sa->runnable_avg_sum, periods + 1); @@ -4023,7 +4025,7 @@ static inline void hrtick_update(struct rq *rq) } #endif -static unsigned int capacity_margin = 1280; /* ~20% margin */ +unsigned int __read_mostly sysctl_sched_capacity_margin = 1280; /* ~20% margin */ static bool cpu_overutilized(int cpu); static unsigned long get_cpu_usage(int cpu); @@ -4096,7 +4098,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) unsigned long req_cap = get_boosted_cpu_usage(cpu_of(rq)); - req_cap = req_cap * capacity_margin + req_cap = req_cap * sysctl_sched_capacity_margin >> SCHED_CAPACITY_SHIFT; cpufreq_sched_set_cap(cpu_of(rq), req_cap); } @@ -4179,7 +4181,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) get_boosted_cpu_usage(cpu_of(rq)); if (rq->cfs.nr_running) { - req_cap = req_cap * capacity_margin + req_cap = req_cap * sysctl_sched_capacity_margin >> SCHED_CAPACITY_SHIFT; cpufreq_sched_set_cap(cpu_of(rq), req_cap); } else { @@ -4585,7 +4587,7 @@ static int find_new_capacity(struct energy_env *eenv, static bool cpu_overutilized(int cpu) { return (capacity_of(cpu) * 1024) < - (get_cpu_usage(cpu) * capacity_margin); + (get_cpu_usage(cpu) * sysctl_sched_capacity_margin); } static int group_idle_state(struct sched_group *sg) @@ -5014,10 +5016,13 @@ boosted_task_utilization(struct task_struct *task) static inline bool __task_fits(struct task_struct *p, int cpu, int usage) { unsigned long capacity = capacity_of(cpu); + int orig_usage = usage; usage += boosted_task_utilization(p); - return (capacity * 1024) > (usage * capacity_margin); + trace_sched_task_fits(p, cpu, orig_usage, usage, task_utilization(p), capacity, sysctl_sched_capacity_margin); + + return (capacity * 1024) > (usage * sysctl_sched_capacity_margin); } static inline bool task_fits_capacity(struct task_struct *p, int cpu) @@ -5028,7 +5033,7 @@ static inline bool task_fits_capacity(struct task_struct *p, int cpu) if (capacity == max_capacity) return true; - if (capacity * capacity_margin > max_capacity * 1024) + if (capacity * sysctl_sched_capacity_margin > max_capacity * 1024) return true; return __task_fits(p, cpu, 0); @@ -5102,7 +5107,7 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, struct sched_group *fit_group = NULL, *spare_group = NULL; unsigned long min_load = ULONG_MAX, this_load = 0; unsigned long fit_capacity = ULONG_MAX; - unsigned long max_spare_capacity = capacity_margin - SCHED_LOAD_SCALE; + unsigned long max_spare_capacity = sysctl_sched_capacity_margin - SCHED_LOAD_SCALE; int load_idx = sd->forkexec_idx; int imbalance = 100 + (sd->imbalance_pct-100)/2; @@ -6805,7 +6810,7 @@ group_is_overloaded(struct lb_env *env, struct sg_lb_stats *sgs) static inline bool group_smaller_cpu_capacity(struct sched_group *sg, struct sched_group *ref) { - return sg->sgc->max_capacity + capacity_margin - SCHED_LOAD_SCALE < + return sg->sgc->max_capacity + sysctl_sched_capacity_margin - SCHED_LOAD_SCALE < ref->sgc->max_capacity; } @@ -7642,7 +7647,7 @@ static int load_balance(int this_cpu, struct rq *this_rq, unsigned long req_cap = get_boosted_cpu_usage(env.src_cpu); - req_cap = req_cap * capacity_margin + req_cap = req_cap * sysctl_sched_capacity_margin >> SCHED_CAPACITY_SHIFT; cpufreq_sched_set_cap(env.src_cpu, req_cap); } @@ -7671,7 +7676,7 @@ static int load_balance(int this_cpu, struct rq *this_rq, unsigned long req_cap = get_boosted_cpu_usage(env.dst_cpu); - req_cap = req_cap * capacity_margin + req_cap = req_cap * sysctl_sched_capacity_margin >> SCHED_CAPACITY_SHIFT; cpufreq_sched_set_cap(env.dst_cpu, req_cap); } @@ -8047,7 +8052,7 @@ static int active_load_balance_cpu_stop(void *data) unsigned long req_cap = get_boosted_cpu_usage(env.src_cpu); - req_cap = req_cap * capacity_margin + req_cap = req_cap * sysctl_sched_capacity_margin >> SCHED_CAPACITY_SHIFT; cpufreq_sched_set_cap(env.src_cpu, req_cap); } @@ -8073,7 +8078,7 @@ static int active_load_balance_cpu_stop(void *data) unsigned long req_cap = get_boosted_cpu_usage(target_cpu); - req_cap = req_cap * capacity_margin + req_cap = req_cap * sysctl_sched_capacity_margin >> SCHED_CAPACITY_SHIFT; cpufreq_sched_set_cap(target_cpu, req_cap); } @@ -8556,7 +8561,7 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) if (capacity_curr < capacity_orig && (capacity_curr * SCHED_LOAD_SCALE) < - (get_cpu_usage(cpu) * capacity_margin)) + (get_cpu_usage(cpu) * sysctl_sched_capacity_margin)) cpufreq_sched_set_cap(cpu, capacity_orig); } } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index ed2201b362e70..23ce100129cac 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1289,6 +1289,7 @@ static inline int idle_get_state_idx(struct rq *rq) } #endif +#if 0 #ifndef arch_scale_cpu_capacity static __always_inline unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu) @@ -1299,6 +1300,7 @@ unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu) return SCHED_CAPACITY_SCALE; } #endif +#endif unsigned long capacity_orig_of(int cpu); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 1333338c98875..93e79b936328e 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -297,8 +297,8 @@ static struct ctl_table kern_table[] = { }, #endif { - .procname = "sched_wakeup_load_threshold", - .data = &sysctl_sched_wakeup_load_threshold, + .procname = "sched_capacity_margin", + .data = &sysctl_sched_capacity_margin, .maxlen = sizeof(unsigned int), .mode = 0644, .proc_handler = proc_dointvec, From 8227ddc0004c435a51956ef628dda3b86e6efc22 Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Tue, 22 Dec 2015 09:47:06 -0800 Subject: [PATCH 17/86] Make energy_aware_wake smarter about cstates and overutilization Change-Id: Iae592f82f6ead6c3c42817c134b695b841920c4b --- kernel/sched/fair.c | 139 ++++++++++++++++++++++++++------------------ 1 file changed, 82 insertions(+), 57 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index a40447dc21c0a..1aa1f5a383228 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5280,89 +5280,114 @@ static int select_idle_sibling(struct task_struct *p, int target) sg = sg->next; } while (sg != sd->groups); } + done: return target; } static int energy_aware_wake_cpu(struct task_struct *p, int target) { + int prev_cpu = task_cpu(p); struct sched_domain *sd; - struct sched_group *sg, *sg_target; - int target_max_cap = INT_MAX; - int target_cpu = task_cpu(p); - int i; - - sd = rcu_dereference(per_cpu(sd_ea, task_cpu(p))); + struct sched_group *sg; + int i = prev_cpu; + int best_busy = -1; + int best_idle = -1; + int best_idle_cstate = -1; + int best_busy_load = INT_MAX; + int better_busy = -1; + int better_idle = -1; + int better_idle_cstate = -1; + int better_busy_load = INT_MAX; - if (!sd) + if (idle_cpu(target)) return target; - sg = sd->groups; - sg_target = sg; - /* - * Find group with sufficient capacity. We only get here if no cpu is - * overutilized. We may end up overutilizing a cpu by adding the task, - * but that should not be any worse than select_idle_sibling(). - * load_balance() should sort it out later as we get above the tipping - * point. + * If the prevous cpu is cache affine and idle, don't be stupid. */ - do { - /* Assuming all cpus are the same in group */ - int max_cap_cpu = group_first_cpu(sg); - - /* - * Assume smaller max capacity means more energy-efficient. - * Ideally we should query the energy model for the right - * answer but it easily ends up in an exhaustive search. - */ - if (capacity_of(max_cap_cpu) < target_max_cap && - task_fits_capacity(p, max_cap_cpu)) { - sg_target = sg; - target_max_cap = capacity_of(max_cap_cpu); - } - } while (sg = sg->next, sg != sd->groups); - - /* Find cpu with sufficient capacity */ - for_each_cpu_and(i, tsk_cpus_allowed(p), sched_group_cpus(sg_target)) { - /* - * p's blocked utilization is still accounted for on prev_cpu - * so prev_cpu will receive a negative bias due the double - * accouting. However, the blocked utilization may be zero. - */ - int new_usage = get_cpu_usage(i) + boosted_task_utilization(p); + if (i != target && cpus_share_cache(i, target) && idle_cpu(i)) + return i; - if (new_usage > capacity_orig_of(i)) - continue; + /* + * Otherwise, iterate the domains and find an elegible idle cpu. + */ + sd = rcu_dereference(per_cpu(sd_llc, target)); + for_each_lower_domain(sd) { + sg = sd->groups; + do { + if (!cpumask_intersects(sched_group_cpus(sg), + tsk_cpus_allowed(p))) + goto next; - if (new_usage < capacity_curr_of(i)) { - target_cpu = i; - if (cpu_rq(i)->nr_running) - break; - } + for_each_cpu(i, sched_group_cpus(sg)) { + struct rq *rq = cpu_rq(i); + int idx = idle_get_state_idx(rq); + unsigned long usage = get_cpu_usage(i); + unsigned long new_usage = usage + boosted_task_utilization(p); + if (new_usage < capacity_curr_of(i)) { + /* Task fits here -- use it if not idle*/ + if (!idx) goto done; + } - /* cpu has capacity at higher OPP, keep it as fallback */ - if (target_cpu == task_cpu(p)) - target_cpu = i; + if (new_usage <= capacity_orig_of(i)) { + /* Task fits -- best case idle/busy */ + if (idx && (best_idle < 0 || + idx < best_idle_cstate)) { + best_idle = i; + best_idle_cstate = idx; + } else if (best_busy < 0 || + best_busy_load > usage) { + /* Task fits, but requires higher OPP */ + best_busy = i; + best_busy_load = usage; + } + } else { + /* Task doesn't fit, but keep track of + * better idle/busy. + */ + if (idx && (better_idle < 0 || + idx < better_idle_cstate)) { + better_idle = i; + better_idle_cstate = idx; + } else if (better_busy < 0 || + better_busy_load > usage) { + better_busy = i; + better_busy_load = usage; + } + } + } +next: + sg = sg->next; + } while (sg != sd->groups); } - - if (target_cpu != task_cpu(p)) { + if (best_idle >= 0) + target = best_idle; + else if (best_busy >= 0) + target = best_busy; + else if (better_idle >= 0) + target = better_idle; + else if (better_busy >= 0) + target = better_busy; + /* else use passed-in target */ +done: + if (target != prev_cpu) { struct energy_env eenv = { .usage_delta = task_utilization(p), .src_cpu = task_cpu(p), - .dst_cpu = target_cpu, + .dst_cpu = target, .task = p, }; /* Not enough spare capacity on previous cpu */ - if (cpu_overutilized(task_cpu(p))) - return target_cpu; + if (cpu_overutilized(prev_cpu)) + return target; if (energy_diff(&eenv) >= 0) - return task_cpu(p); + return prev_cpu; } - return target_cpu; + return target; } /* @@ -5420,7 +5445,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f prev_cpu = cpu; if (sd_flag & SD_BALANCE_WAKE && want_sibling) { - if (energy_aware() && !cpu_rq(cpu)->rd->overutilized) + if (energy_aware()) new_cpu = energy_aware_wake_cpu(p, prev_cpu); else new_cpu = select_idle_sibling(p, prev_cpu); From b667212840a951e5539157a92f5f10f5e734b873 Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Mon, 4 Jan 2016 16:14:25 -0800 Subject: [PATCH 18/86] Added instramentation and tuned energy_aware_wake_cpu() Change-Id: I70fb1b40d86fd37a94e2cea39b63b1fcb5519894 --- kernel/sched/cpufreq_sched.c | 2 + kernel/sched/fair.c | 181 +++++++++++++++++++++++++++++------ 2 files changed, 152 insertions(+), 31 deletions(-) diff --git a/kernel/sched/cpufreq_sched.c b/kernel/sched/cpufreq_sched.c index c067462dc3402..7129ee88d1656 100644 --- a/kernel/sched/cpufreq_sched.c +++ b/kernel/sched/cpufreq_sched.c @@ -369,6 +369,8 @@ static int cpufreq_sched_setup(struct cpufreq_policy *policy, unsigned int event /* Tunables */ static ssize_t show_throttle_ns(struct gov_data *gd, char *buf) { + extern void tk_print(void); + tk_print(); return sprintf(buf, "%u\n", gd->throttle_nsec); } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 1aa1f5a383228..27c6a52746aba 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5285,12 +5285,84 @@ static int select_idle_sibling(struct task_struct *p, int target) return target; } -static int energy_aware_wake_cpu(struct task_struct *p, int target) -{ - int prev_cpu = task_cpu(p); +int tk_is_idle = 0; +int tk_no_stupid = 0; +int tk_best_busy = 0; +int tk_best_idle = 0; +int tk_better_busy =0; +int tk_better_idle = 0; +int tk_bb_check = 0; +int tk_bb_wins = 0; +int tk_bb_loses = 0; +int tk_ediff_override = 0; +int tk_target = 0; +int tk_ea_wake = 0; +int tk_diff = 0; +int tk_over = 0; +int tk_use_over = 0; +int tk_sync = 0; +int tk_usage = 0; +int tk_fits = 0; + +void tk_clear(void) { + tk_is_idle = 0; + tk_no_stupid = 0; + tk_best_busy = 0; + tk_best_idle = 0; + tk_better_busy =0; + tk_better_idle = 0; + tk_bb_check = 0; + tk_bb_wins = 0; + tk_bb_loses = 0; + tk_ediff_override = 0; + tk_target = 0; + tk_ea_wake = 0; + tk_diff = 0; + tk_over = 0; + tk_use_over = 0; + tk_sync = 0; + tk_usage = 0; + tk_fits = 0; +} + +#define TKPCNT(_c, _t) \ + if (_t != 0) { \ + int _pct = (100*(_c)) / (_t); \ + pr_info("TJK: " #_c "=%d / %d (%d%%)\n", _c, _t, _pct); \ + } else { \ + pr_info("TJK: " #_c "=%d / %d\n", _c, _t); \ + } + + +void tk_print(void) { + TKPCNT(tk_is_idle, tk_ea_wake); + TKPCNT(tk_no_stupid, tk_ea_wake); + TKPCNT(tk_use_over, tk_ea_wake); + TKPCNT(tk_best_busy, tk_ea_wake); + TKPCNT(tk_best_idle, tk_ea_wake); + TKPCNT(tk_better_busy, tk_ea_wake); + TKPCNT(tk_better_idle, tk_ea_wake); + TKPCNT(tk_bb_check, tk_ea_wake); + TKPCNT(tk_bb_wins, tk_bb_check); + TKPCNT(tk_bb_loses, tk_bb_check); + TKPCNT(tk_ediff_override, tk_ea_wake); + TKPCNT(tk_target, tk_ea_wake); + TKPCNT(tk_diff, tk_ea_wake); + TKPCNT(tk_sync, tk_ea_wake); + TKPCNT(tk_usage, tk_ea_wake); + TKPCNT(tk_fits, tk_ea_wake); + TKPCNT(tk_over, tk_ea_wake); + tk_clear(); + tk_clear(); +} + + +static int energy_aware_wake_cpu(struct task_struct *p, int target, bool sync) +{ + int default_cpu = task_cpu(p); struct sched_domain *sd; struct sched_group *sg; - int i = prev_cpu; + int i = default_cpu; int best_busy = -1; int best_idle = -1; int best_idle_cstate = -1; @@ -5299,20 +5371,23 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target) int better_idle = -1; int better_idle_cstate = -1; int better_busy_load = INT_MAX; + int best_idle_capacity = INT_MAX; + int best_busy_capacity = INT_MAX; + bool force_over = false; + tk_ea_wake++; - if (idle_cpu(target)) - return target; - - /* - * If the prevous cpu is cache affine and idle, don't be stupid. - */ - if (i != target && cpus_share_cache(i, target) && idle_cpu(i)) - return i; + if (sync) { + tk_sync++; // 12% + goto done; + } /* * Otherwise, iterate the domains and find an elegible idle cpu. */ sd = rcu_dereference(per_cpu(sd_llc, target)); + if (!sd) + return target; + for_each_lower_domain(sd) { sg = sd->groups; do { @@ -5321,26 +5396,34 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target) goto next; for_each_cpu(i, sched_group_cpus(sg)) { + struct rq *rq = cpu_rq(i); int idx = idle_get_state_idx(rq); unsigned long usage = get_cpu_usage(i); unsigned long new_usage = usage + boosted_task_utilization(p); - if (new_usage < capacity_curr_of(i)) { - /* Task fits here -- use it if not idle*/ - if (!idx) goto done; + unsigned long capacity_orig = capacity_orig_of(i); + unsigned long capacity_curr = capacity_curr_of(i); + + if (new_usage <= capacity_curr) { + if (i == target) { + tk_usage++; // 76% + goto done; + } } - if (new_usage <= capacity_orig_of(i)) { + if (new_usage <= capacity_orig) { /* Task fits -- best case idle/busy */ if (idx && (best_idle < 0 || - idx < best_idle_cstate)) { + (idx < best_idle_cstate && capacity_orig <= best_idle_capacity))) { best_idle = i; best_idle_cstate = idx; + best_idle_capacity = capacity_orig; } else if (best_busy < 0 || - best_busy_load > usage) { + (best_busy_load > usage && capacity_orig <= best_busy_capacity)) { /* Task fits, but requires higher OPP */ best_busy = i; best_busy_load = usage; + best_busy_capacity = capacity_orig; } } else { /* Task doesn't fit, but keep track of @@ -5361,32 +5444,68 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target) sg = sg->next; } while (sg != sd->groups); } - if (best_idle >= 0) + + if (best_idle >= 0) { + /* Let energy calculation decide + * between best_busy and best_idle + */ + tk_use_over++; // 2% + force_over = true; target = best_idle; - else if (best_busy >= 0) - target = best_busy; - else if (better_idle >= 0) - target = better_idle; - else if (better_busy >= 0) - target = better_busy; + if (best_busy >= 0) + default_cpu = best_busy; // 2% + else { + tk_best_idle++; // 0% + return best_idle; + } + } else if (best_busy >= 0) { + tk_best_busy++; // 1% + return best_busy; + } + else if (better_idle >= 0) { + tk_better_idle++; // 6% + return better_idle; + } + else if (better_busy >= 0) { + tk_better_busy++; // 0% + return better_busy; + } /* else use passed-in target */ done: - if (target != prev_cpu) { + if (target != default_cpu) { struct energy_env eenv = { .usage_delta = task_utilization(p), - .src_cpu = task_cpu(p), + .src_cpu = default_cpu, .dst_cpu = target, .task = p, }; + tk_diff++; /* Not enough spare capacity on previous cpu */ - if (cpu_overutilized(prev_cpu)) + if (cpu_overutilized(default_cpu)) { + tk_over++; // 0% return target; + } - if (energy_diff(&eenv) >= 0) - return prev_cpu; + if (default_cpu != task_cpu(p)) { + tk_bb_check++; // 1% + if (energy_diff(&eenv) >= 0) { + tk_bb_wins++; + return default_cpu; + } + else { + tk_bb_loses++; + return target; + } + } + + if (force_over && energy_diff(&eenv) >= 0) { + tk_ediff_override++; // 0% + return default_cpu; + } } + tk_target++; // 88% return target; } @@ -5446,7 +5565,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f if (sd_flag & SD_BALANCE_WAKE && want_sibling) { if (energy_aware()) - new_cpu = energy_aware_wake_cpu(p, prev_cpu); + new_cpu = energy_aware_wake_cpu(p, prev_cpu, sync); else new_cpu = select_idle_sibling(p, prev_cpu); goto unlock; From 730b4acf456050a0c965d99cc27a093632fbe392 Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Tue, 5 Jan 2016 15:35:05 -0800 Subject: [PATCH 19/86] Tune energy_aware_wake_cpu() / select_idle_sibling() - don't assume cpus are loaded evenly - be aware of cstates when selecting idle cpu - try to assign current cpu if sync wakeup Change-Id: I509a547186c826cc451f6b8d2ed58bb4a868ec49 --- kernel/sched/cpufreq_sched.c | 2 - kernel/sched/fair.c | 298 +++++++++++------------------------ 2 files changed, 91 insertions(+), 209 deletions(-) diff --git a/kernel/sched/cpufreq_sched.c b/kernel/sched/cpufreq_sched.c index 7129ee88d1656..c067462dc3402 100644 --- a/kernel/sched/cpufreq_sched.c +++ b/kernel/sched/cpufreq_sched.c @@ -369,8 +369,6 @@ static int cpufreq_sched_setup(struct cpufreq_policy *policy, unsigned int event /* Tunables */ static ssize_t show_throttle_ns(struct gov_data *gd, char *buf) { - extern void tk_print(void); - tk_print(); return sprintf(buf, "%u\n", gd->throttle_nsec); } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 27c6a52746aba..2df3766854d7a 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -37,7 +37,6 @@ #include "tune.h" #ifndef TJK_HMP -// TJK: resolve undefined sysctl symbols unsigned int __read_mostly sysctl_sched_wake_to_idle = 0; #endif @@ -5247,15 +5246,19 @@ static int select_idle_sibling(struct task_struct *p, int target) struct sched_domain *sd; struct sched_group *sg; int i = task_cpu(p); + int best_idle = -1; + int best_idle_cstate = -1; + int best_idle_capacity = INT_MAX; +#if 0 if (idle_cpu(target)) return target; - /* * If the prevous cpu is cache affine and idle, don't be stupid. */ if (i != target && cpus_share_cache(i, target) && idle_cpu(i)) return i; +#endif /* * Otherwise, iterate the domains and find an elegible idle cpu. @@ -5268,245 +5271,127 @@ static int select_idle_sibling(struct task_struct *p, int target) tsk_cpus_allowed(p))) goto next; - for_each_cpu(i, sched_group_cpus(sg)) { - if (i == target || !idle_cpu(i)) + for_each_cpu_and(i, tsk_cpus_allowed(p), sched_group_cpus(sg)) { + struct rq *rq = cpu_rq(i); + int idle_idx = idle_get_state_idx(rq); + unsigned long new_usage = boosted_task_utilization(p); + unsigned long capacity_orig = capacity_orig_of(i); + if (new_usage > capacity_orig || !idle_cpu(i)) goto next; + + if (i == target && new_usage <= capacity_curr_of(target)) + return target; + + if (best_idle < 0 || + (idle_idx < best_idle_cstate && capacity_orig <= best_idle_capacity)) { + best_idle = i; + best_idle_cstate = idle_idx; + best_idle_capacity = capacity_orig; + } } - target = cpumask_first_and(sched_group_cpus(sg), - tsk_cpus_allowed(p)); - goto done; next: sg = sg->next; } while (sg != sd->groups); } -done: + if (best_idle >= 0) + target = best_idle; + return target; } -int tk_is_idle = 0; -int tk_no_stupid = 0; -int tk_best_busy = 0; -int tk_best_idle = 0; -int tk_better_busy =0; -int tk_better_idle = 0; -int tk_bb_check = 0; -int tk_bb_wins = 0; -int tk_bb_loses = 0; -int tk_ediff_override = 0; -int tk_target = 0; -int tk_ea_wake = 0; -int tk_diff = 0; -int tk_over = 0; -int tk_use_over = 0; -int tk_sync = 0; -int tk_usage = 0; -int tk_fits = 0; - -void tk_clear(void) { - tk_is_idle = 0; - tk_no_stupid = 0; - tk_best_busy = 0; - tk_best_idle = 0; - tk_better_busy =0; - tk_better_idle = 0; - tk_bb_check = 0; - tk_bb_wins = 0; - tk_bb_loses = 0; - tk_ediff_override = 0; - tk_target = 0; - tk_ea_wake = 0; - tk_diff = 0; - tk_over = 0; - tk_use_over = 0; - tk_sync = 0; - tk_usage = 0; - tk_fits = 0; -} - -#define TKPCNT(_c, _t) \ - if (_t != 0) { \ - int _pct = (100*(_c)) / (_t); \ - pr_info("TJK: " #_c "=%d / %d (%d%%)\n", _c, _t, _pct); \ - } else { \ - pr_info("TJK: " #_c "=%d / %d\n", _c, _t); \ - } - - -void tk_print(void) { - TKPCNT(tk_is_idle, tk_ea_wake); - TKPCNT(tk_no_stupid, tk_ea_wake); - TKPCNT(tk_use_over, tk_ea_wake); - TKPCNT(tk_best_busy, tk_ea_wake); - TKPCNT(tk_best_idle, tk_ea_wake); - TKPCNT(tk_better_busy, tk_ea_wake); - TKPCNT(tk_better_idle, tk_ea_wake); - TKPCNT(tk_bb_check, tk_ea_wake); - TKPCNT(tk_bb_wins, tk_bb_check); - TKPCNT(tk_bb_loses, tk_bb_check); - TKPCNT(tk_ediff_override, tk_ea_wake); - TKPCNT(tk_target, tk_ea_wake); - TKPCNT(tk_diff, tk_ea_wake); - TKPCNT(tk_sync, tk_ea_wake); - TKPCNT(tk_usage, tk_ea_wake); - TKPCNT(tk_fits, tk_ea_wake); - TKPCNT(tk_over, tk_ea_wake); - tk_clear(); - tk_clear(); -} - - -static int energy_aware_wake_cpu(struct task_struct *p, int target, bool sync) -{ - int default_cpu = task_cpu(p); +static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync) +{ struct sched_domain *sd; - struct sched_group *sg; - int i = default_cpu; - int best_busy = -1; - int best_idle = -1; - int best_idle_cstate = -1; - int best_busy_load = INT_MAX; - int better_busy = -1; - int better_idle = -1; - int better_idle_cstate = -1; - int better_busy_load = INT_MAX; - int best_idle_capacity = INT_MAX; - int best_busy_capacity = INT_MAX; - bool force_over = false; - tk_ea_wake++; + struct sched_group *sg, *sg_target; + int target_max_cap = INT_MAX; + int target_cpu = -1; + int i; if (sync) { - tk_sync++; // 12% - goto done; + int cpu = smp_processor_id(); + cpumask_t search_cpus; + cpumask_and(&search_cpus, tsk_cpus_allowed(p), cpu_online_mask); + if (cpumask_test_cpu(cpu, &search_cpus)) + return cpu; } - /* - * Otherwise, iterate the domains and find an elegible idle cpu. - */ - sd = rcu_dereference(per_cpu(sd_llc, target)); + sd = rcu_dereference(per_cpu(sd_ea, task_cpu(p))); + if (!sd) return target; - for_each_lower_domain(sd) { - sg = sd->groups; - do { - if (!cpumask_intersects(sched_group_cpus(sg), - tsk_cpus_allowed(p))) - goto next; + sg = sd->groups; + sg_target = sg; - for_each_cpu(i, sched_group_cpus(sg)) { + /* + * Find group with sufficient capacity. We only get here if no cpu is + * overutilized. We may end up overutilizing a cpu by adding the task, + * but that should not be any worse than select_idle_sibling(). + * load_balance() should sort it out later as we get above the tipping + * point. + */ + do { + /* + * Assume smaller max capacity means more energy-efficient. + * Ideally we should query the energy model for the right + * answer but it easily ends up in an exhaustive search. + */ + for_each_cpu_and(i, tsk_cpus_allowed(p), sched_group_cpus(sg)) { + + if (capacity_of(i) < target_max_cap && + task_fits_capacity(p, i)) { + sg_target = sg; + target_max_cap = capacity_of(i); + } else if (cpu_rq(i)->rd->overutilized) + return select_idle_sibling(p, target); + } - struct rq *rq = cpu_rq(i); - int idx = idle_get_state_idx(rq); - unsigned long usage = get_cpu_usage(i); - unsigned long new_usage = usage + boosted_task_utilization(p); - unsigned long capacity_orig = capacity_orig_of(i); - unsigned long capacity_curr = capacity_curr_of(i); + } while (sg = sg->next, sg != sd->groups); - if (new_usage <= capacity_curr) { - if (i == target) { - tk_usage++; // 76% - goto done; - } - } + /* Find cpu with sufficient capacity */ + for_each_cpu_and(i, tsk_cpus_allowed(p), sched_group_cpus(sg_target)) { + /* + * p's blocked utilization is still accounted for on prev_cpu + * so prev_cpu will receive a negative bias due the double + * accouting. However, the blocked utilization may be zero. + */ + int new_usage = get_cpu_usage(i) + boosted_task_utilization(p); - if (new_usage <= capacity_orig) { - /* Task fits -- best case idle/busy */ - if (idx && (best_idle < 0 || - (idx < best_idle_cstate && capacity_orig <= best_idle_capacity))) { - best_idle = i; - best_idle_cstate = idx; - best_idle_capacity = capacity_orig; - } else if (best_busy < 0 || - (best_busy_load > usage && capacity_orig <= best_busy_capacity)) { - /* Task fits, but requires higher OPP */ - best_busy = i; - best_busy_load = usage; - best_busy_capacity = capacity_orig; - } - } else { - /* Task doesn't fit, but keep track of - * better idle/busy. - */ - if (idx && (better_idle < 0 || - idx < better_idle_cstate)) { - better_idle = i; - better_idle_cstate = idx; - } else if (better_busy < 0 || - better_busy_load > usage) { - better_busy = i; - better_busy_load = usage; - } - } - } -next: - sg = sg->next; - } while (sg != sd->groups); - } + if (new_usage > capacity_orig_of(i)) + continue; - if (best_idle >= 0) { - /* Let energy calculation decide - * between best_busy and best_idle - */ - tk_use_over++; // 2% - force_over = true; - target = best_idle; - if (best_busy >= 0) - default_cpu = best_busy; // 2% - else { - tk_best_idle++; // 0% - return best_idle; + if (new_usage < capacity_curr_of(i)) { + target_cpu = i; + if (cpu_rq(i)->nr_running) + break; } - } else if (best_busy >= 0) { - tk_best_busy++; // 1% - return best_busy; - } - else if (better_idle >= 0) { - tk_better_idle++; // 6% - return better_idle; - } - else if (better_busy >= 0) { - tk_better_busy++; // 0% - return better_busy; + + /* cpu has capacity at higher OPP, keep it as fallback */ + if (target_cpu < 0) + target_cpu = i; } - /* else use passed-in target */ -done: - if (target != default_cpu) { + + if (target_cpu < 0) + target_cpu = task_cpu(p); + else if (target_cpu != task_cpu(p)) { struct energy_env eenv = { .usage_delta = task_utilization(p), - .src_cpu = default_cpu, - .dst_cpu = target, + .src_cpu = task_cpu(p), + .dst_cpu = target_cpu, .task = p, }; - tk_diff++; /* Not enough spare capacity on previous cpu */ - if (cpu_overutilized(default_cpu)) { - tk_over++; // 0% - return target; - } - - if (default_cpu != task_cpu(p)) { - tk_bb_check++; // 1% - if (energy_diff(&eenv) >= 0) { - tk_bb_wins++; - return default_cpu; - } - else { - tk_bb_loses++; - return target; - } - } + if (cpu_overutilized(task_cpu(p))) + return target_cpu; - if (force_over && energy_diff(&eenv) >= 0) { - tk_ediff_override++; // 0% - return default_cpu; - } + if (energy_diff(&eenv) >= 0) + return task_cpu(p); } - tk_target++; // 88% - return target; + return target_cpu; } /* @@ -7710,7 +7595,6 @@ static int load_balance(int this_cpu, struct rq *this_rq, struct sched_group *group; struct rq *busiest; unsigned long flags; - //struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); struct cpumask *cpus = this_cpu_ptr(load_balance_mask); struct lb_env env = { From dfb8ef9b3eeecdc4b5e6a0f49f775dd4d9361b93 Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Fri, 8 Jan 2016 14:00:06 -0800 Subject: [PATCH 20/86] Fix handling of max frequency If policy->max is change, the new frequency calculation will be incorrect. Keep track of the highest maximum since it corresponds to the max capacity for the CPU. Still an issue if the cpu is onlined with a reduced policy->max. The frequency calcs will be incorrect until policy->max is raised to the maximum value Change-Id: Ib2cfbc8efe2c56ea0568792745c4b0591a294ae6 --- kernel/sched/cpufreq_sched.c | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/kernel/sched/cpufreq_sched.c b/kernel/sched/cpufreq_sched.c index c067462dc3402..cd965f066cce6 100644 --- a/kernel/sched/cpufreq_sched.c +++ b/kernel/sched/cpufreq_sched.c @@ -41,6 +41,7 @@ struct gov_data { unsigned int freq; bool change_pending; struct list_head gov_list; + int max; }; /* worker thread for dvfs transition that may block/sleep */ @@ -146,6 +147,7 @@ static void cpufreq_sched_irq_work(struct irq_work *irq_work) * 1) this cpu did not the new maximum capacity for its frequency domain * 2) no change in cpu frequency is necessary to meet the new capacity request */ + void cpufreq_sched_set_cap(int cpu, unsigned long capacity) { unsigned int freq_new, cpu_tmp; @@ -190,7 +192,13 @@ void cpufreq_sched_set_cap(int cpu, unsigned long capacity) goto out; /* Convert the new maximum capacity request into a cpu frequency */ - freq_new = (capacity * policy->max) / capacity_orig_of(cpu); + freq_new = (capacity * gd->max) / capacity_orig_of(cpu); + + if (freq_new > policy->max) + freq_new = policy->max; + + if (freq_new < policy->min) + freq_new = policy->min; /* No change in frequency? Bail and return current capacity. */ if (freq_new == policy->cur) @@ -308,6 +316,7 @@ static int cpufreq_sched_policy_init(struct cpufreq_policy *policy) policy->governor_data = gd; gd->policy = policy; + gd->max = policy->max; mutex_lock(&gov_list_lock); list_add_tail(&gd->gov_list, &sched_gov_list); @@ -351,6 +360,7 @@ static int cpufreq_sched_policy_exit(struct cpufreq_policy *policy) static int cpufreq_sched_setup(struct cpufreq_policy *policy, unsigned int event) { + struct gov_data *gd; switch (event) { case CPUFREQ_GOV_START: return cpufreq_sched_policy_start(policy); @@ -360,7 +370,24 @@ static int cpufreq_sched_setup(struct cpufreq_policy *policy, unsigned int event return cpufreq_sched_policy_init(policy); case CPUFREQ_GOV_POLICY_EXIT: return cpufreq_sched_policy_exit(policy); - case CPUFREQ_GOV_LIMITS: /* unused */ + case CPUFREQ_GOV_LIMITS: + mutex_lock(&gov_list_lock); + pr_debug("limit event for cpu %u: %u - %u kHz, currently %u kHz\n", + policy->cpu, policy->min, policy->max, + policy->cur); + /* + * Need to keep track of highest max frequency for + * capacity calculations + */ + gd = policy->governor_data; + if (gd->max < policy->max) + gd->max = policy->max; + + if (policy->max < policy->cur) + __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H); + else if (policy->min > policy->cur) + __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L); + mutex_unlock(&gov_list_lock); break; } return 0; From c725d05b3971dd1596f8fbdb13b6b4fe3b474852 Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Fri, 8 Jan 2016 14:02:47 -0800 Subject: [PATCH 21/86] Revert earlier change to handling overutilized cpu Change-Id: Ifc79d176468d0028e9be2ae630bb865fd18d9f8b --- kernel/sched/fair.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 2df3766854d7a..067934fee8a65 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5333,19 +5333,18 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync) * point. */ do { + /* Assuming all cpus are the same in group */ + int max_cap_cpu = group_first_cpu(sg); + /* * Assume smaller max capacity means more energy-efficient. * Ideally we should query the energy model for the right * answer but it easily ends up in an exhaustive search. */ - for_each_cpu_and(i, tsk_cpus_allowed(p), sched_group_cpus(sg)) { - - if (capacity_of(i) < target_max_cap && - task_fits_capacity(p, i)) { + if (capacity_of(max_cap_cpu) < target_max_cap && + task_fits_capacity(p, max_cap_cpu)) { sg_target = sg; - target_max_cap = capacity_of(i); - } else if (cpu_rq(i)->rd->overutilized) - return select_idle_sibling(p, target); + target_max_cap = capacity_of(max_cap_cpu); } } while (sg = sg->next, sg != sd->groups); @@ -5449,7 +5448,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f prev_cpu = cpu; if (sd_flag & SD_BALANCE_WAKE && want_sibling) { - if (energy_aware()) + if (energy_aware() && !cpu_rq(cpu)->rd->overutilized) new_cpu = energy_aware_wake_cpu(p, prev_cpu, sync); else new_cpu = select_idle_sibling(p, prev_cpu); From 288b8efdf0dd57fc4f3cc2e5e21352cfaed75652 Mon Sep 17 00:00:00 2001 From: Kirill Tkhai Date: Mon, 27 Oct 2014 17:40:52 +0300 Subject: [PATCH 22/86] UPSTREAM: sched/deadline: Implement cancel_dl_timer() to use in switched_from_dl() Currently used hrtimer_try_to_cancel() is racy: raw_spin_lock(&rq->lock) ... dl_task_timer raw_spin_lock(&rq->lock) ... raw_spin_lock(&rq->lock) ... switched_from_dl() ... ... hrtimer_try_to_cancel() ... ... switched_to_fair() ... ... ... ... ... ... ... ... raw_spin_unlock(&rq->lock) ... (asquired) ... ... ... ... ... ... do_exit() ... ... schedule() ... ... raw_spin_lock(&rq->lock) ... raw_spin_unlock(&rq->lock) ... ... ... raw_spin_unlock(&rq->lock) ... raw_spin_lock(&rq->lock) ... ... (asquired) put_task_struct() ... ... free_task_struct() ... ... ... ... raw_spin_unlock(&rq->lock) ... (asquired) ... ... ... ... ... (use after free) ... So, let's implement 100% guaranteed way to cancel the timer and let's be sure we are safe even in very unlikely situations. rq unlocking does not limit the area of switched_from_dl() use, because this has already been possible in pull_dl_task() below. Let's consider the safety of of this unlocking. New code in the patch is working when hrtimer_try_to_cancel() fails. This means the callback is running. In this case hrtimer_cancel() is just waiting till the callback is finished. Two 1) Since we are in switched_from_dl(), new class is not dl_sched_class and new prio is not less MAX_DL_PRIO. So, the callback returns early; it's right after !dl_task() check. After that hrtimer_cancel() returns back too. The above is: raw_spin_lock(rq->lock); ... ... dl_task_timer() ... raw_spin_lock(rq->lock); switched_from_dl() ... hrtimer_try_to_cancel() ... raw_spin_unlock(rq->lock); ... hrtimer_cancel() ... ... raw_spin_unlock(rq->lock); ... return HRTIMER_NORESTART; ... ... raw_spin_lock(rq->lock); ... 2) But the below is also possible: dl_task_timer() raw_spin_lock(rq->lock); ... raw_spin_unlock(rq->lock); raw_spin_lock(rq->lock); ... switched_from_dl() ... hrtimer_try_to_cancel() ... ... return HRTIMER_NORESTART; raw_spin_unlock(rq->lock); ... hrtimer_cancel(); ... raw_spin_lock(rq->lock); ... In this case hrtimer_cancel() returns immediately. Very unlikely case, just to mention. Nobody can manipulate the task, because check_class_changed() is always called with pi_lock locked. Nobody can force the task to participate in (concurrent) priority inheritance schemes (the same reason). All concurrent task operations require pi_lock, which is held by us. No deadlocks with dl_task_timer() are possible, because it returns right after !dl_task() check (it does nothing). If we receive a new dl_task during the time of unlocked rq, we just don't have to do pull_dl_task() in switched_from_dl() further. Signed-off-by: Kirill Tkhai [ Added comments] Signed-off-by: Peter Zijlstra (Intel) Acked-by: Juri Lelli Cc: Linus Torvalds Link: http://lkml.kernel.org/r/1414420852.19914.186.camel@tkhai Signed-off-by: Ingo Molnar (cherry picked from commit 67dfa1b756f250972bde31d65e3f8fde6aeddc5b) Signed-off-by: Punit Agrawal BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: I3c5a4d39d6408c3bc9d516746098edd0edb944b5 Reviewed-on: https://chromium-review.googlesource.com/321251 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/core.c | 4 ++++ kernel/sched/deadline.c | 34 +++++++++++++++++++++++++++------- kernel/sched/sched.h | 5 +++++ 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 1f341763ea909..41595c490dec6 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1013,6 +1013,9 @@ inline int task_curr(const struct task_struct *p) return cpu_curr(task_cpu(p)) == p; } +/* + * Can drop rq->lock because from sched_class::switched_from() methods drop it. + */ static inline void check_class_changed(struct rq *rq, struct task_struct *p, const struct sched_class *prev_class, int oldprio) @@ -1020,6 +1023,7 @@ static inline void check_class_changed(struct rq *rq, struct task_struct *p, if (prev_class != p->sched_class) { if (prev_class->switched_from) prev_class->switched_from(rq, p); + /* Possble rq->lock 'hole'. */ p->sched_class->switched_to(rq, p); } else if (oldprio != p->prio || dl_task(p)) p->sched_class->prio_changed(rq, p, oldprio); diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index fc0fd03698610..c78b6bc59747b 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -563,11 +563,6 @@ void init_dl_task_timer(struct sched_dl_entity *dl_se) { struct hrtimer *timer = &dl_se->dl_timer; - if (hrtimer_active(timer)) { - hrtimer_try_to_cancel(timer); - return; - } - hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); timer->function = dl_task_timer; } @@ -1569,10 +1564,35 @@ void init_sched_dl_class(void) #endif /* CONFIG_SMP */ +/* + * Ensure p's dl_timer is cancelled. May drop rq->lock for a while. + */ +static void cancel_dl_timer(struct rq *rq, struct task_struct *p) +{ + struct hrtimer *dl_timer = &p->dl.dl_timer; + + /* Nobody will change task's class if pi_lock is held */ + lockdep_assert_held(&p->pi_lock); + + if (hrtimer_active(dl_timer)) { + int ret = hrtimer_try_to_cancel(dl_timer); + + if (unlikely(ret == -1)) { + /* + * Note, p may migrate OR new deadline tasks + * may appear in rq when we are unlocking it. + * A caller of us must be fine with that. + */ + raw_spin_unlock(&rq->lock); + hrtimer_cancel(dl_timer); + raw_spin_lock(&rq->lock); + } + } +} + static void switched_from_dl(struct rq *rq, struct task_struct *p) { - if (hrtimer_active(&p->dl.dl_timer) && !dl_policy(p->policy)) - hrtimer_try_to_cancel(&p->dl.dl_timer); + cancel_dl_timer(rq, p); __dl_clear_params(p); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 23ce100129cac..0f29b3cce72c8 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1198,6 +1198,11 @@ struct sched_class { void (*task_fork) (struct task_struct *p); void (*task_dead) (struct task_struct *p); + /* + * The switched_from() call is allowed to drop rq->lock, therefore we + * cannot assume the switched_from/switched_to pair is serliazed by + * rq->lock. They are however serialized by p->pi_lock. + */ void (*switched_from) (struct rq *this_rq, struct task_struct *task); void (*switched_to) (struct rq *this_rq, struct task_struct *task); void (*prio_changed) (struct rq *this_rq, struct task_struct *task, From 80eb61e6e9f73fd9a8ed7e8112a70e4eb63c5aeb Mon Sep 17 00:00:00 2001 From: Wanpeng Li Date: Fri, 31 Oct 2014 06:39:32 +0800 Subject: [PATCH 23/86] UPSTREAM: sched/deadline: Fix artificial overrun introduced by yield_task_dl() The yield semantic of deadline class is to reduce remaining runtime to zero, and then update_curr_dl() will stop it. However, comsumed bandwidth is reduced from the budget of yield task again even if it has already been set to zero which leads to artificial overrun. This patch fix it by make sure we don't steal some more time from the task that yielded in update_curr_dl(). Suggested-by: Juri Lelli Signed-off-by: Wanpeng Li Signed-off-by: Peter Zijlstra (Intel) Cc: Kirill Tkhai Cc: Linus Torvalds Link: http://lkml.kernel.org/r/1414708776-124078-2-git-send-email-wanpeng.li@linux.intel.com Signed-off-by: Ingo Molnar (cherry picked from commit 804968809c321066cca028d4cbd533a420f964bc) Signed-off-by: Punit Agrawal BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: I9e098e742847eb6282848d4ee474bc451a8c978b Reviewed-on: https://chromium-review.googlesource.com/321252 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/deadline.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index c78b6bc59747b..a7e461f9a4e89 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -611,7 +611,7 @@ static void update_curr_dl(struct rq *rq) sched_rt_avg_update(rq, delta_exec); - dl_se->runtime -= delta_exec; + dl_se->runtime -= dl_se->dl_yielded ? 0 : delta_exec; if (dl_runtime_exceeded(rq, dl_se)) { __dequeue_task_dl(rq, curr, 0); if (likely(start_dl_timer(dl_se, curr->dl.dl_boosted))) From be33a9f47041aa3b1bc495ac8caa448ca3c9f281 Mon Sep 17 00:00:00 2001 From: Wanpeng Li Date: Fri, 31 Oct 2014 06:39:35 +0800 Subject: [PATCH 24/86] UPSTREAM: sched/deadline: Reschedule from switched_from_dl() after a successful pull In switched_from_dl() we have to issue a resched if we successfully pulled some task from other cpus. This patch also aligns the behavior with -rt. Suggested-by: Juri Lelli Signed-off-by: Wanpeng Li Signed-off-by: Peter Zijlstra (Intel) Cc: Kirill Tkhai Cc: Linus Torvalds Link: http://lkml.kernel.org/r/1414708776-124078-5-git-send-email-wanpeng.li@linux.intel.com Signed-off-by: Ingo Molnar (cherry picked from commit cd66091162d34f589631a23bbe0ed214798245b4) Signed-off-by: Punit Agrawal BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: Icddd7a8fb29f5d96d7b25f710c89f4df7dc31ae6 Reviewed-on: https://chromium-review.googlesource.com/321253 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/deadline.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index a7e461f9a4e89..5527def24d0cb 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -1602,8 +1602,11 @@ static void switched_from_dl(struct rq *rq, struct task_struct *p) * this is the right place to try to pull some other one * from an overloaded cpu, if any. */ - if (!rq->dl.dl_nr_running) - pull_dl_task(rq); + if (!task_on_rq_queued(p) || rq->dl.dl_nr_running) + return; + + if (pull_dl_task(rq)) + resched_curr(rq); #endif } From 13419a3b188f1de348af1452d4ecfba660e6cdcd Mon Sep 17 00:00:00 2001 From: Wanpeng Li Date: Fri, 31 Oct 2014 06:39:36 +0800 Subject: [PATCH 25/86] UPSTREAM: sched/deadline: Don't check CONFIG_SMP in switched_from_dl() There are both UP and SMP version of pull_dl_task(), so don't need to check CONFIG_SMP in switched_from_dl(); Signed-off-by: Wanpeng Li Signed-off-by: Peter Zijlstra (Intel) Cc: Juri Lelli Cc: Kirill Tkhai Cc: Linus Torvalds Link: http://lkml.kernel.org/r/1414708776-124078-6-git-send-email-wanpeng.li@linux.intel.com Signed-off-by: Ingo Molnar (cherry picked from commit cad3bb32e181c286c46ec12b2deb1f26a6f9835d) Signed-off-by: Punit Agrawal BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: I439e0c22ca0754104035cb249b6fe2d8fcbbf5d6 Reviewed-on: https://chromium-review.googlesource.com/321254 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/deadline.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 5527def24d0cb..9fca2abb8e8fa 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -1596,7 +1596,6 @@ static void switched_from_dl(struct rq *rq, struct task_struct *p) __dl_clear_params(p); -#ifdef CONFIG_SMP /* * Since this might be the only -deadline task on the rq, * this is the right place to try to pull some other one @@ -1607,7 +1606,6 @@ static void switched_from_dl(struct rq *rq, struct task_struct *p) if (pull_dl_task(rq)) resched_curr(rq); -#endif } /* From b14320260cb4422773ce664612d59ab804bde5f0 Mon Sep 17 00:00:00 2001 From: Kirill Tkhai Date: Mon, 15 Dec 2014 14:56:58 +0300 Subject: [PATCH 26/86] UPSTREAM: sched/fair: Fix sched_entity::avg::decay_count initialization Child has the same decay_count as parent. If it's not zero, we add it to parent's cfs_rq->removed_load: wake_up_new_task()->set_task_cpu()->migrate_task_rq_fair(). Child's load is a just garbade after copying of parent, it hasn't been on cfs_rq yet, and it must not be added to cfs_rq::removed_load in migrate_task_rq_fair(). The patch moves sched_entity::avg::decay_count intialization in sched_fork(). So, migrate_task_rq_fair() does not change removed_load. Signed-off-by: Kirill Tkhai Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Ben Segall Cc: Linus Torvalds Link: http://lkml.kernel.org/r/1418644618.6074.13.camel@tkhai Signed-off-by: Ingo Molnar (cherry picked from commit bb04159df99fa353d0fb524574aca03ce2c6515b) Signed-off-by: Ricky Liang BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: I1eae9d4612040b449727d73289effd61a1ea493e Reviewed-on: https://chromium-review.googlesource.com/312182 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/core.c | 3 +++ kernel/sched/fair.c | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 41595c490dec6..d2d50062d013a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1864,6 +1864,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) p->se.prev_sum_exec_runtime = 0; p->se.nr_migrations = 0; p->se.vruntime = 0; +#ifdef CONFIG_SMP + p->se.avg.decay_count = 0; +#endif INIT_LIST_HEAD(&p->se.group_node); #ifdef CONFIG_SCHEDSTATS diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 067934fee8a65..0e00b098c0c6b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -696,11 +696,11 @@ static inline void __update_task_entity_utilization(struct sched_entity *se); /* Give new task start runnable values to heavy its load in infant time */ void init_task_runnable_average(struct task_struct *p) { - u32 start_load = sysctl_sched_latency >> 10; + u32 slice; - p->se.avg.decay_count = 0; - p->se.avg.runnable_avg_sum = p->se.avg.running_avg_sum = start_load; - p->se.avg.avg_period = start_load; + slice = sched_slice(task_cfs_rq(p), &p->se) >> 10; + p->se.avg.runnable_avg_sum = p->se.avg.running_avg_sum = slice; + p->se.avg.avg_period = slice; __update_task_entity_contrib(&p->se); __update_task_entity_utilization(&p->se); } From 017183966f50f62bb678569bce02d5e56788b6ce Mon Sep 17 00:00:00 2001 From: Yao Dongdong Date: Mon, 29 Dec 2014 14:41:43 +0800 Subject: [PATCH 27/86] UPSTREAM: sched/core: Remove check of p->sched_class Search all usage of p->sched_class in sched/core.c, no one check it before use, so it seems that every task must belong to one sched_class. Signed-off-by: Yao Dongdong [ Moved the early class assignment to make it boot. ] Signed-off-by: Peter Zijlstra (Intel) Cc: Linus Torvalds Link: http://lkml.kernel.org/r/1419835303-28958-1-git-send-email-yaodongdong@huawei.com Signed-off-by: Ingo Molnar (cherry picked from commit 1b537c7d1e58c761212a193085f9049b58f672e6) Signed-off-by: Ricky Liang BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: I5fa325b8657b7c1f976224ca3a2a174d3fbce660 Reviewed-on: https://chromium-review.googlesource.com/321503 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/core.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d2d50062d013a..d3ba16bd33636 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4801,7 +4801,7 @@ static struct rq *move_queued_task(struct task_struct *p, int new_cpu) void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) { - if (p->sched_class && p->sched_class->set_cpus_allowed) + if (p->sched_class->set_cpus_allowed) p->sched_class->set_cpus_allowed(p, new_mask); cpumask_copy(&p->cpus_allowed, new_mask); @@ -7434,6 +7434,11 @@ void __init sched_init(void) atomic_inc(&init_mm.mm_count); enter_lazy_tlb(&init_mm, current); + /* + * During early bootup we pretend to be a normal task: + */ + current->sched_class = &fair_sched_class; + /* * Make us the idle thread. Technically, schedule() should not be * called from this thread, however somewhere below it might be, @@ -7444,11 +7449,6 @@ void __init sched_init(void) calc_load_update = jiffies + LOAD_FREQ; - /* - * During early bootup we pretend to be a normal task: - */ - current->sched_class = &fair_sched_class; - #ifdef CONFIG_SMP zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT); /* May be allocated at isolcpus cmdline parse time */ From 376044712f2cdcd235169382305a17f5fe4fb441 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 5 Jan 2015 11:18:10 +0100 Subject: [PATCH 28/86] UPSTREAM: sched/core: Validate rq_clock*() serialization rq->clock{,_task} are serialized by rq->lock, verify this. One immediate fail is the usage in scale_rt_capability, so 'annotate' that for now, there's more 'funny' there. Maybe change rq->lock into a raw_seqlock_t? (Only 32-bit is affected) Signed-off-by: Peter Zijlstra (Intel) Link: http://lkml.kernel.org/r/20150105103554.361872747@infradead.org Cc: Linus Torvalds Cc: umgwanakikbuti@gmail.com Signed-off-by: Ingo Molnar (cherry picked from commit cebde6d681aa45f96111cfcffc1544cf2a0454ff) Signed-off-by: Ricky Liang BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: If36b2c5765242bd9de224d5d552b2b3ad61f1942 Reviewed-on: https://chromium-review.googlesource.com/312185 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/fair.c | 2 +- kernel/sched/sched.h | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 0e00b098c0c6b..1491c9d41937d 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6638,8 +6638,8 @@ static unsigned long scale_rt_capacity(int cpu) */ age_stamp = ACCESS_ONCE(rq->age_stamp); avg = ACCESS_ONCE(rq->rt_avg); + delta = __rq_clock_broken(rq) - age_stamp; - delta = rq_clock(rq) - age_stamp; if (unlikely(delta < 0)) delta = 0; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 0f29b3cce72c8..bc64c52e11712 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -702,13 +702,20 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); #define cpu_curr(cpu) (cpu_rq(cpu)->curr) #define raw_rq() raw_cpu_ptr(&runqueues) +static inline u64 __rq_clock_broken(struct rq *rq) +{ + return ACCESS_ONCE(rq->clock); +} + static inline u64 rq_clock(struct rq *rq) { + lockdep_assert_held(&rq->lock); return rq->clock; } static inline u64 rq_clock_task(struct rq *rq) { + lockdep_assert_held(&rq->lock); return rq->clock_task; } From 72b7c7c6cd9bcfc84a04dcf3e10aae1bae277e5c Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 28 Jan 2015 15:08:03 +0100 Subject: [PATCH 29/86] UPSTREAM: sched/deadline: Fix deadline parameter modification handling Commit 67dfa1b756f2 ("sched/deadline: Implement cancel_dl_timer() to use in switched_from_dl()") removed the hrtimer_try_cancel() function call out from init_dl_task_timer(), which gets called from __setparam_dl(). The result is that we can now re-init the timer while its active -- this is bad and corrupts timer state. Furthermore; changing the parameters of an active deadline task is tricky in that you want to maintain guarantees, while immediately effective change would allow one to circumvent the CBS guarantees -- this too is bad, as one (bad) task should not be able to affect the others. Rework things to avoid both problems. We only need to initialize the timer once, so move that to __sched_fork() for new tasks. Then make sure __setparam_dl() doesn't affect the current running state but only updates the parameters used to calculate the next scheduling period -- this guarantees the CBS functions as expected (albeit slightly pessimistic). This however means we need to make sure __dl_clear_params() needs to reset the active state otherwise new (and tasks flipping between classes) will not properly (re)compute their first instance. Todo: close class flipping CBS hole. Todo: implement delayed BW release. Reported-by: Luca Abeni Acked-by: Juri Lelli Tested-by: Luca Abeni Fixes: 67dfa1b756f2 ("sched/deadline: Implement cancel_dl_timer() to use in switched_from_dl()") Signed-off-by: Peter Zijlstra (Intel) Cc: Cc: Kirill Tkhai Cc: Linus Torvalds Link: http://lkml.kernel.org/r/20150128140803.GF23038@twins.programming.kicks-ass.net Signed-off-by: Ingo Molnar (cherry picked from commit 40767b0dc768060266d261b4a330164b4be53f7c) Signed-off-by: Punit Agrawal BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: I1341d4a4c891d6215eea6d249f95324e79c923e5 Reviewed-on: https://chromium-review.googlesource.com/321256 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/core.c | 33 ++++++++++++++++++++++++++++----- kernel/sched/deadline.c | 3 ++- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d3ba16bd33636..8ab3f18ea4517 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1846,6 +1846,10 @@ void __dl_clear_params(struct task_struct *p) dl_se->dl_period = 0; dl_se->flags = 0; dl_se->dl_bw = 0; + + dl_se->dl_throttled = 0; + dl_se->dl_new = 1; + dl_se->dl_yielded = 0; } /* @@ -1874,7 +1878,7 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) #endif RB_CLEAR_NODE(&p->dl.rb_node); - hrtimer_init(&p->dl.dl_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + init_dl_task_timer(&p->dl); __dl_clear_params(p); INIT_LIST_HEAD(&p->rt.run_list); @@ -2105,6 +2109,9 @@ bool __dl_overflow(struct dl_bw *dl_b, int cpus, u64 old_bw, u64 new_bw) * allocated bandwidth to reflect the new situation. * * This function is called while holding p's rq->lock. + * + * XXX we should delay bw change until the task's 0-lag point, see + * __setparam_dl(). */ static int dl_overflow(struct task_struct *p, int policy, const struct sched_attr *attr) @@ -3347,15 +3354,31 @@ __setparam_dl(struct task_struct *p, const struct sched_attr *attr) { struct sched_dl_entity *dl_se = &p->dl; - init_dl_task_timer(dl_se); dl_se->dl_runtime = attr->sched_runtime; dl_se->dl_deadline = attr->sched_deadline; dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline; dl_se->flags = attr->sched_flags; dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); - dl_se->dl_throttled = 0; - dl_se->dl_new = 1; - dl_se->dl_yielded = 0; + + /* + * Changing the parameters of a task is 'tricky' and we're not doing + * the correct thing -- also see task_dead_dl() and switched_from_dl(). + * + * What we SHOULD do is delay the bandwidth release until the 0-lag + * point. This would include retaining the task_struct until that time + * and change dl_overflow() to not immediately decrement the current + * amount. + * + * Instead we retain the current runtime/deadline and let the new + * parameters take effect after the current reservation period lapses. + * This is safe (albeit pessimistic) because the 0-lag point is always + * before the current scheduling deadline. + * + * We can still have temporary overloads because we do not delay the + * change in bandwidth until that time; so admission control is + * not on the safe side. It does however guarantee tasks will never + * consume more than promised. + */ } /* diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 9fca2abb8e8fa..0fe896b3e0222 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -1094,6 +1094,7 @@ static void task_dead_dl(struct task_struct *p) * Since we are TASK_DEAD we won't slip out of the domain! */ raw_spin_lock_irq(&dl_b->lock); + /* XXX we should retain the bw until 0-lag */ dl_b->total_bw -= p->dl.dl_bw; raw_spin_unlock_irq(&dl_b->lock); @@ -1592,8 +1593,8 @@ static void cancel_dl_timer(struct rq *rq, struct task_struct *p) static void switched_from_dl(struct rq *rq, struct task_struct *p) { + /* XXX we should retain the bw until 0-lag */ cancel_dl_timer(rq, p); - __dl_clear_params(p); /* From 8c087ab45ddf8cd0bf619bed58d623c124d7825a Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 26 Nov 2014 08:44:03 +0800 Subject: [PATCH 30/86] UPSTREAM: sched/deadline: Fix stale yield state When we fail to start the deadline timer in update_curr_dl(), we forget to clear ->dl_yielded, resulting in wrecked time keeping. Since the natural place to clear both ->dl_yielded and ->dl_throttled is in replenish_dl_entity(); both are after all waiting for that event; make it so. Luckily since 67dfa1b756f2 ("sched/deadline: Implement cancel_dl_timer() to use in switched_from_dl()") the task_on_rq_queued() condition in dl_task_timer() must be true, and can therefore call enqueue_task_dl() unconditionally. Reported-by: Wanpeng Li Signed-off-by: Peter Zijlstra (Intel) Cc: Kirill Tkhai Cc: Juri Lelli Cc: Linus Torvalds Link: http://lkml.kernel.org/r/1416962647-76792-4-git-send-email-wanpeng.li@linux.intel.com Signed-off-by: Ingo Molnar (cherry picked from commit 1019a359d3dc4b64d0e1e5a5efcb725d5e83994d) Signed-off-by: Punit Agrawal BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: Ib1518aea1a87519c4adb5789080f31d2fe09c580 Reviewed-on: https://chromium-review.googlesource.com/321257 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/deadline.c | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 0fe896b3e0222..7e12797ed42aa 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -350,6 +350,11 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se, dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; dl_se->runtime = pi_se->dl_runtime; } + + if (dl_se->dl_yielded) + dl_se->dl_yielded = 0; + if (dl_se->dl_throttled) + dl_se->dl_throttled = 0; } /* @@ -536,23 +541,19 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) sched_clock_tick(); update_rq_clock(rq); - dl_se->dl_throttled = 0; - dl_se->dl_yielded = 0; - if (task_on_rq_queued(p)) { - enqueue_task_dl(rq, p, ENQUEUE_REPLENISH); - if (dl_task(rq->curr)) - check_preempt_curr_dl(rq, p, 0); - else - resched_curr(rq); + enqueue_task_dl(rq, p, ENQUEUE_REPLENISH); + if (dl_task(rq->curr)) + check_preempt_curr_dl(rq, p, 0); + else + resched_curr(rq); #ifdef CONFIG_SMP - /* - * Queueing this task back might have overloaded rq, - * check if we need to kick someone away. - */ - if (has_pushable_dl_tasks(rq)) - push_dl_task(rq); + /* + * Queueing this task back might have overloaded rq, + * check if we need to kick someone away. + */ + if (has_pushable_dl_tasks(rq)) + push_dl_task(rq); #endif - } unlock: raw_spin_unlock(&rq->lock); @@ -613,10 +614,9 @@ static void update_curr_dl(struct rq *rq) dl_se->runtime -= dl_se->dl_yielded ? 0 : delta_exec; if (dl_runtime_exceeded(rq, dl_se)) { + dl_se->dl_throttled = 1; __dequeue_task_dl(rq, curr, 0); - if (likely(start_dl_timer(dl_se, curr->dl.dl_boosted))) - dl_se->dl_throttled = 1; - else + if (unlikely(!start_dl_timer(dl_se, curr->dl.dl_boosted))) enqueue_task_dl(rq, curr, ENQUEUE_REPLENISH); if (!is_leftmost(curr, &rq->dl)) @@ -853,7 +853,7 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) * its rq, the bandwidth timer callback (which clearly has not * run yet) will take care of this. */ - if (p->dl.dl_throttled) + if (p->dl.dl_throttled && !(flags & ENQUEUE_REPLENISH)) return; enqueue_dl_entity(&p->dl, pi_se, flags); From 40a833805171e89a8b560ee662d131af6b4f7370 Mon Sep 17 00:00:00 2001 From: Kirill Tkhai Date: Mon, 16 Feb 2015 15:38:34 +0300 Subject: [PATCH 31/86] UPSTREAM: sched/dl: Prevent enqueue of a sleeping task in dl_task_timer() A deadline task may be throttled and dequeued at the same time. This happens, when it becomes throttled in schedule(), which is called to go to sleep: current->state = TASK_INTERRUPTIBLE; schedule() deactivate_task() dequeue_task_dl() update_curr_dl() start_dl_timer() __dequeue_task_dl() prev->on_rq = 0; Later the timer fires, but the task is still dequeued: dl_task_timer() enqueue_task_dl() /* queues on dl_rq; on_rq remains 0 */ Someone wakes it up: try_to_wake_up() enqueue_dl_entity() BUG_ON(on_dl_rq()) Patch fixes this problem, it prevents queueing !on_rq tasks on dl_rq. Reported-by: Fengguang Wu Signed-off-by: Kirill Tkhai Signed-off-by: Peter Zijlstra (Intel) [ Wrote comment. ] Cc: Juri Lelli Fixes: 1019a359d3dc ("sched/deadline: Fix stale yield state") Link: http://lkml.kernel.org/r/1374601424090314@web4j.yandex.ru Signed-off-by: Ingo Molnar (cherry picked from commit a79ec89fd8459f0de850898f432a2a57d60e64de) Signed-off-by: Punit Agrawal BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: I9095c05ad17fda0ba75230f08dcd11a5458fd8bf Reviewed-on: https://chromium-review.googlesource.com/321258 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/deadline.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 7e12797ed42aa..2e126d9070f30 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -541,6 +541,26 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) sched_clock_tick(); update_rq_clock(rq); + + /* + * If the throttle happened during sched-out; like: + * + * schedule() + * deactivate_task() + * dequeue_task_dl() + * update_curr_dl() + * start_dl_timer() + * __dequeue_task_dl() + * prev->on_rq = 0; + * + * We can be both throttled and !queued. Replenish the counter + * but do not enqueue -- wait for our wakeup to do that. + */ + if (!task_on_rq_queued(p)) { + replenish_dl_entity(dl_se, dl_se); + goto unlock; + } + enqueue_task_dl(rq, p, ENQUEUE_REPLENISH); if (dl_task(rq->curr)) check_preempt_curr_dl(rq, p, 0); From e1aa3168443b6f712b914b942b59ea12dfdacf53 Mon Sep 17 00:00:00 2001 From: Wanpeng Li Date: Fri, 27 Mar 2015 07:08:35 +0800 Subject: [PATCH 32/86] UPSTREAM: sched/deadline: Support DL task migration during CPU hotplug I observed that DL tasks can't be migrated to other CPUs during CPU hotplug, in addition, task may/may not be running again if CPU is added back. The root cause which I found is that DL tasks will be throtted and removed from the DL rq after comsuming all their budget, which leads to the situation that stop task can't pick them up from the DL rq and migrate them to other CPUs during hotplug. The method to reproduce: schedtool -E -t 50000:100000 -e ./test Actually './test' is just a simple for loop. Then observe which CPU the test task is on and offline it: echo 0 > /sys/devices/system/cpu/cpuN/online This patch adds the DL task migration during CPU hotplug by finding a most suitable later deadline rq after DL timer fires if current rq is offline. If it fails to find a suitable later deadline rq then it falls back to any eligible online CPU in so that the deadline task will come back to us, and the push/pull mechanism should then move it around properly. Suggested-and-Acked-by: Juri Lelli Signed-off-by: Wanpeng Li Signed-off-by: Peter Zijlstra (Intel) Link: http://lkml.kernel.org/r/1427411315-4298-1-git-send-email-wanpeng.li@linux.intel.com Signed-off-by: Ingo Molnar (cherry picked from commit fa9c9d10e97e38d9903fad1829535175ad261f45) Signed-off-by: Punit Agrawal BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: I3825ce5dfdf574a088270a5043caeacebc3e8927 Reviewed-on: https://chromium-review.googlesource.com/321259 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/deadline.c | 57 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 2e126d9070f30..201d122fbeac4 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -218,6 +218,52 @@ static inline void set_post_schedule(struct rq *rq) rq->post_schedule = has_pushable_dl_tasks(rq); } +static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq); + +static void dl_task_offline_migration(struct rq *rq, struct task_struct *p) +{ + struct rq *later_rq = NULL; + bool fallback = false; + + later_rq = find_lock_later_rq(p, rq); + + if (!later_rq) { + int cpu; + + /* + * If we cannot preempt any rq, fall back to pick any + * online cpu. + */ + fallback = true; + cpu = cpumask_any_and(cpu_active_mask, tsk_cpus_allowed(p)); + if (cpu >= nr_cpu_ids) { + /* + * Fail to find any suitable cpu. + * The task will never come back! + */ + BUG_ON(dl_bandwidth_enabled()); + + /* + * If admission control is disabled we + * try a little harder to let the task + * run. + */ + cpu = cpumask_any(cpu_active_mask); + } + later_rq = cpu_rq(cpu); + double_lock_balance(rq, later_rq); + } + + deactivate_task(rq, p, 0); + set_task_cpu(p, later_rq->cpu); + activate_task(later_rq, p, ENQUEUE_REPLENISH); + + if (!fallback) + resched_curr(later_rq); + + double_unlock_balance(rq, later_rq); +} + #else static inline @@ -542,6 +588,17 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) sched_clock_tick(); update_rq_clock(rq); +#ifdef CONFIG_SMP + /* + * If we find that the rq the task was on is no longer + * available, we need to select a new rq. + */ + if (unlikely(!rq->online)) { + dl_task_offline_migration(rq, p); + goto unlock; + } +#endif + /* * If the throttle happened during sched-out; like: * From 14b52e4fb7301ad33d2cb22fd0c1a0bb50b4b0fc Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 14 Apr 2015 21:09:06 +0000 Subject: [PATCH 33/86] UPSTREAM: sched: deadline: Use hrtimer_start() hrtimer_start() does not longer defer already expired timers to the softirq. Get rid of the __hrtimer_start_range_ns() invocation. Signed-off-by: Thomas Gleixner Acked-by: Peter Zijlstra Cc: Preeti U Murthy Cc: Viresh Kumar Cc: Marcelo Tosatti Cc: Frederic Weisbecker Link: http://lkml.kernel.org/r/20150414203502.627353666@linutronix.de Signed-off-by: Thomas Gleixner (cherry picked from commit cc9684d3c1188ac5f1cf0ee9f8be7ba456099d7b) Signed-off-by: Punit Agrawal BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: I86e6bfe59a35b718c4fd99991943dc705f887ef5 Reviewed-on: https://chromium-review.googlesource.com/321260 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/deadline.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 201d122fbeac4..72af5cd760393 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -503,8 +503,6 @@ static int start_dl_timer(struct sched_dl_entity *dl_se, bool boosted) struct dl_rq *dl_rq = dl_rq_of_se(dl_se); struct rq *rq = rq_of_dl_rq(dl_rq); ktime_t now, act; - ktime_t soft, hard; - unsigned long range; s64 delta; if (boosted) @@ -527,15 +525,9 @@ static int start_dl_timer(struct sched_dl_entity *dl_se, bool boosted) if (ktime_us_delta(act, now) < 0) return 0; - hrtimer_set_expires(&dl_se->dl_timer, act); + hrtimer_start(&dl_se->dl_timer, act, HRTIMER_MODE_ABS); - soft = hrtimer_get_softexpires(&dl_se->dl_timer); - hard = hrtimer_get_expires(&dl_se->dl_timer); - range = ktime_to_ns(ktime_sub(hard, soft)); - __hrtimer_start_range_ns(&dl_se->dl_timer, soft, - range, HRTIMER_MODE_ABS, 0); - - return hrtimer_active(&dl_se->dl_timer); + return 1; } /* From fac6e38f90bba789161e7736ba957a467514e8e7 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 31 Oct 2013 18:07:08 +0100 Subject: [PATCH 34/86] sched: Move wait.c into kernel/sched/ Suggested-by: Ingo Molnar Signed-off-by: Peter Zijlstra Cc: Linus Torvalds Cc: Andrew Morton Link: http://lkml.kernel.org/n/tip-5q5yqvdaen0rmapwloeaotx3@git.kernel.org Signed-off-by: Ingo Molnar --- kernel/Makefile | 2 +- kernel/sched/Makefile | 2 +- kernel/{ => sched}/wait.c | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename kernel/{ => sched}/wait.c (100%) diff --git a/kernel/Makefile b/kernel/Makefile index 99d1fec9da3dc..fac636723f59f 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -7,7 +7,7 @@ obj-y = fork.o exec_domain.o panic.o printk.o \ sysctl.o sysctl_binary.o capability.o ptrace.o timer.o user.o \ signal.o sys.o kmod.o workqueue.o pid.o task_work.o \ rcupdate.o extable.o params.o posix-timers.o \ - kthread.o wait.o sys_ni.o posix-cpu-timers.o mutex.o \ + kthread.o sys_ni.o posix-cpu-timers.o mutex.o \ hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \ notifier.o ksysfs.o cred.o \ async.o range.o groups.o lglock.o smpboot.o diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index ea6e4df4f860e..3ba40adebbc6a 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -13,7 +13,7 @@ endif obj-y += core.o proc.o clock.o cputime.o obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o energy.o -obj-y += completion.o +obj-y += completion.o wait.o obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o obj-$(CONFIG_SCHEDSTATS) += stats.o diff --git a/kernel/wait.c b/kernel/sched/wait.c similarity index 100% rename from kernel/wait.c rename to kernel/sched/wait.c From ca151a802fa857985050a7046df1bfed124449a1 Mon Sep 17 00:00:00 2001 From: Robin Randhawa Date: Mon, 29 Jun 2015 18:01:58 +0100 Subject: [PATCH 35/86] CHROMIUM: sched: Support for extracting EAS energy costs from DT This patch implements support for extracting energy cost data from DT. The data should conform to the DT bindings for energy cost data needed by EAS (energy aware scheduling). BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. Change-Id: Ia435bd4d4b111bb6257ffb2f5385b5f4b70d5aa6 Signed-off-by: Robin Randhawa Signed-off-by: Ricky Liang From 9279fac2c5fbf6bfe7814d2e71afa09a28c2b5cf Mon Sep 17 00:00:00 2001 From: Robin Randhawa Date: Mon, 29 Jun 2015 17:56:20 +0100 Subject: [PATCH 36/86] CHROMIUM: Documentation: DT bindings for energy model cost data required by EAS EAS (energy aware scheduling) provides the scheduler with an alternative objective - energy efficiency - as opposed to it's current performance oriented objectives. EAS relies on a simple platform energy cost model to guide scheduling decisions. The model only considers the CPU subsystem. This patch adds documentation describing DT bindings that should be used to supply the scheduler with an energy cost model. BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. Change-Id: I312c8d2f46d3aed0b8f39bd6e4f1739699bc5944 Signed-off-by: Robin Randhawa Signed-off-by: Ricky Liang --- .../bindings/scheduler/sched-energy-costs.txt | 360 ++++++++++++++++++ 1 file changed, 360 insertions(+) create mode 100644 Documentation/devicetree/bindings/scheduler/sched-energy-costs.txt diff --git a/Documentation/devicetree/bindings/scheduler/sched-energy-costs.txt b/Documentation/devicetree/bindings/scheduler/sched-energy-costs.txt new file mode 100644 index 0000000000000..11216f09e596b --- /dev/null +++ b/Documentation/devicetree/bindings/scheduler/sched-energy-costs.txt @@ -0,0 +1,360 @@ +=========================================================== +Energy cost bindings for Energy Aware Scheduling +=========================================================== + +=========================================================== +1 - Introduction +=========================================================== + +This note specifies bindings required for energy-aware scheduling +(EAS)[1]. Historically, the scheduler's primary objective has been +performance. EAS aims to provide an alternative objective - energy +efficiency. EAS relies on a simple platform energy cost model to +guide scheduling decisions. The model only considers the CPU +subsystem. + +This note is aligned with the definition of the layout of physical +CPUs in the system as described in the ARM topology binding +description [2]. The concept is applicable to any system so long as +the cost model data is provided for those processing elements in +that system's topology that EAS is required to service. + +Processing elements refer to hardware threads, CPUs and clusters of +related CPUs in increasing order of hierarchy. + +EAS requires two key cost metrics - busy costs and idle costs. Busy +costs comprise of a list of compute capacities for the processing +element in question and the corresponding power consumption at that +capacity. Idle costs comprise of a list of power consumption values +for each idle state [C-state] that the processing element supports. +For a detailed description of these metrics, their derivation and +their use see [3]. + +These cost metrics are required for processing elements in all +scheduling domain levels that EAS is required to service. + +=========================================================== +2 - energy-costs node +=========================================================== + +Energy costs for the processing elements in scheduling domains that +EAS is required to service are defined in the energy-costs node +which acts as a container for the actual per processing element cost +nodes. A single energy-costs node is required for a given system. + +- energy-costs node + + Usage: Required + + Description: The energy-costs node is a container node and + it's sub-nodes describe costs for each processing element at + all scheduling domain levels that EAS is required to + service. + + Node name must be "energy-costs". + + The energy-costs node's parent node must be the cpus node. + + The energy-costs node's child nodes can be: + + - one or more cost nodes. + + Any other configuration is considered invalid. + +The energy-costs node can only contain a single type of child node +whose bindings are described in paragraph 4. + +=========================================================== +3 - energy-costs node child nodes naming convention +=========================================================== + +energy-costs child nodes must follow a naming convention where the +node name must be "thread-costN", "core-costN", "cluster-costN" +depending on whether the costs in the node are for a thread, core or +cluster. N (where N = {0, 1, ...}) is the node number and has no +bearing to the OS' logical thread, core or cluster index. + +=========================================================== +4 - cost node bindings +=========================================================== + +Bindings for cost nodes are defined as follows: + +- cluster-cost node + + Description: must be declared within an energy-costs node. A + system can contain multiple clusters and each cluster + serviced by EAS must have a corresponding cluster-costs + node. + + The cluster-cost node name must be "cluster-costN" as + described in 3 above. + + A cluster-cost node must be a leaf node with no children. + + Properties for cluster-cost nodes are described in paragraph + 5 below. + + Any other configuration is considered invalid. + +- core-cost node + + Description: must be declared within an energy-costs node. A + system can contain multiple cores and each core serviced by + EAS must have a corresponding core-cost node. + + The core-cost node name must be "core-costN" as described in + 3 above. + + A core-cost node must be a leaf node with no children. + + Properties for core-cost nodes are described in paragraph + 5 below. + + Any other configuration is considered invalid. + +- thread-cost node + + Description: must be declared within an energy-costs node. A + system can contain cores with multiple hardware threads and + each thread serviced by EAS must have a corresponding + thread-cost node. + + The core-cost node name must be "core-costN" as described in + 3 above. + + A core-cost node must be a leaf node with no children. + + Properties for thread-cost nodes are described in paragraph + 5 below. + + Any other configuration is considered invalid. + +=========================================================== +5 - Cost node properties +========================================================== + +All cost node types must have only the following properties: + +- busy-cost-data + + Usage: required + Value type: An array of 2-item tuples. Each item is of type + u32. + Definition: The first item in the tuple is the capacity + value as described in [3]. The second item in the tuple is + the energy cost value as described in [3]. + +- idle-cost-data + + Usage: required + Value type: An array of 1-item tuples. The item is of type + u32. + Definition: The item in the tuple is the energy cost value + as described in [3]. + +=========================================================== +4 - Extensions to the cpu node +=========================================================== + +The cpu node is extended with a property that establishes the +connection between the processing element represented by the cpu +node and the cost-nodes associated with this processing element. + +The connection is expressed in line with the topological hierarchy +that this processing element belongs to starting with the level in +the hierarchy that this processing element itself belongs to through +to the highest level that EAS is required to service. The +connection cannot be sparse and must be contiguous from the +processing element's level through to the highest desired level. The +highest desired level must be the same for all processing elements. + +Example: Given that a cpu node may represent a thread that is a part +of a core, this property may contain multiple elements which +associate the thread with cost nodes describing the costs for the +thread itself, the core the thread belongs to, the cluster the core +belongs to and so on. The elements must be ordered from the lowest +level nodes to the highest desired level that EAS must service. The +highest desired level must be the same for all cpu nodes. The +elements must not be sparse: there must be elements for the current +thread, the next level of hierarchy (core) and so on without any +'holes'. + +Example: Given that a cpu node may represent a core that is a part +of a cluster of related cpus this property may contain multiple +elements which associate the core with cost nodes describing the +costs for the core itself, the cluster the core belongs to and so +on. The elements must be ordered from the lowest level nodes to the +highest desired level that EAS must service. The highest desired +level must be the same for all cpu nodes. The elements must not be +sparse: there must be elements for the current thread, the next +level of hierarchy (core) and so on without any 'holes'. + +If the system comprises of hierarchical clusters of clusters, this +property will contain multiple associations with the relevant number +of cluster elements in hierarchical order. + +Property added to the cpu node: + +- sched-energy-costs + + Usage: required + Value type: List of phandles + Definition: a list of phandles to specific cost nodes in the + energy-costs parent node that correspond to the processing + element represented by this cpu node in hierarchical order + of topology. + + The order of phandles in the list is significant. The first + phandle is to the current processing element's own cost + node. Subsequent phandles are to higher hierarchical level + cost nodes up until the maximum level that EAS is to + service. + + All cpu nodes must have the same highest level cost node. + + The phandle list must not be sparsely populated with handles + to non-contiguous hierarchical levels. See commentary above + for clarity. + + Any other configuration is invalid. + +=========================================================== +5 - Example dts +=========================================================== + +Example 1 (ARM 64-bit, 6-cpu system, two clusters of cpus, one +cluster of 2 Cortex-A57 cpus, one cluster of 4 Cortex-A53 cpus): + +cpus { + #address-cells = <2>; + #size-cells = <0>; + . + . + . + A57_0: cpu@0 { + compatible = "arm,cortex-a57","arm,armv8"; + reg = <0x0 0x0>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A57_L2>; + clocks = <&scpi_dvfs 0>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + sched-energy-costs = <&CPU_COST_0 &CLUSTER_COST_0>; + }; + + A57_1: cpu@1 { + compatible = "arm,cortex-a57","arm,armv8"; + reg = <0x0 0x1>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A57_L2>; + clocks = <&scpi_dvfs 0>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + sched-energy-costs = <&CPU_COST_0 &CLUSTER_COST_0>; + }; + + A53_0: cpu@100 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x100>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + clocks = <&scpi_dvfs 1>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; + }; + + A53_1: cpu@101 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x101>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + clocks = <&scpi_dvfs 1>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; + }; + + A53_2: cpu@102 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x102>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + clocks = <&scpi_dvfs 1>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; + }; + + A53_3: cpu@103 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x103>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + clocks = <&scpi_dvfs 1>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; + }; + + energy-costs { + CPU_COST_0: core-cost0 { + busy-cost-data = < + 417 168 + 579 251 + 744 359 + 883 479 + 1024 616 + >; + idle-cost-data = < + 15 + 0 + >; + }; + CPU_COST_1: core-cost1 { + busy-cost-data = < + 235 33 + 302 46 + 368 61 + 406 76 + 447 93 + >; + idle-cost-data = < + 6 + 0 + >; + }; + CLUSTER_COST_0: cluster-cost0 { + busy-cost-data = < + 417 24 + 579 32 + 744 43 + 883 49 + 1024 64 + >; + idle-cost-data = < + 65 + 24 + >; + }; + CLUSTER_COST_1: cluster-cost1 { + busy-cost-data = < + 235 26 + 303 30 + 368 39 + 406 47 + 447 57 + >; + idle-cost-data = < + 56 + 17 + >; + }; + }; +}; + +=============================================================================== +[1] https://lkml.org/lkml/2015/5/12/728 +[2] Documentation/devicetree/bindings/topology.txt +[3] Documentation/scheduler/sched-energy.txt From 70145aad118aca4e9359d317393c054a87db9fb5 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 14 Apr 2015 13:19:42 +0200 Subject: [PATCH 37/86] UPSTREAM: sched: Move the loadavg code to a more obvious location I could not find the loadavg code.. turns out it was hidden in a file called proc.c. It further got mingled up with the cruft per rq load indexes (which we really want to get rid of). Move the per rq load indexes into the fair.c load-balance code (that's the only thing that uses them) and rename proc.c to loadavg.c so we can find it again. Signed-off-by: Peter Zijlstra (Intel) Cc: Borislav Petkov Cc: H. Peter Anvin Cc: Paul Gortmaker Cc: Thomas Gleixner [ Did minor cleanups to the code. ] Signed-off-by: Ingo Molnar (cherry picked from commit 3289bdb429884c0279bf9ab72dff7b934f19dfc6) Signed-off-by: Ricky Liang BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: Ib03e7ba34b7ab6c5b40d63d4fb931fc5d5d4fa58 Reviewed-on: https://chromium-review.googlesource.com/312184 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- include/linux/sched.h | 5 + kernel/sched/Makefile | 4 +- kernel/sched/core.c | 7 +- kernel/sched/fair.c | 183 ++++++++++++++++++++++ kernel/sched/{proc.c => loadavg.c} | 236 +++-------------------------- kernel/sched/sched.h | 8 +- 6 files changed, 223 insertions(+), 220 deletions(-) rename kernel/sched/{proc.c => loadavg.c} (62%) diff --git a/include/linux/sched.h b/include/linux/sched.h index 842d9c4c3d5d8..4433afbf30a59 100755 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -179,7 +179,12 @@ extern void sched_update_nr_prod(int cpu, long delta, bool inc); extern void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg); extern void calc_global_load(unsigned long ticks); + +#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) extern void update_cpu_load_nohz(void); +#else +static inline void update_cpu_load_nohz(void) { } +#endif extern unsigned long get_parent_ip(unsigned long addr); diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 3ba40adebbc6a..5a35ff5eebff5 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -11,9 +11,9 @@ ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer endif -obj-y += core.o proc.o clock.o cputime.o +obj-y += core.o loadavg.o clock.o cputime.o obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o energy.o -obj-y += completion.o wait.o +obj-y += wait.o completion.o obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o obj-$(CONFIG_SCHEDSTATS) += stats.o diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 8ab3f18ea4517..25fb2b975a3e2 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2486,9 +2486,9 @@ unsigned long nr_iowait_cpu(int cpu) void get_iowait_load(unsigned long *nr_waiters, unsigned long *load) { - struct rq *this = this_rq(); - *nr_waiters = atomic_read(&this->nr_iowait); - *load = this->cpu_load[0]; + struct rq *rq = this_rq(); + *nr_waiters = atomic_read(&rq->nr_iowait); + *load = rq->load.weight; } #ifdef CONFIG_SMP @@ -2620,6 +2620,7 @@ void scheduler_tick(void) update_rq_clock(rq); curr->sched_class->task_tick(rq, curr, 0); update_cpu_load_active(rq); + calc_global_load_tick(rq); raw_spin_unlock(&rq->lock); perf_event_task_tick(); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 1491c9d41937d..3d9708ff7f645 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4192,6 +4192,189 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) } #ifdef CONFIG_SMP + +/* + * per rq 'load' arrray crap; XXX kill this. + */ + +/* + * The exact cpuload at various idx values, calculated at every tick would be + * load = (2^idx - 1) / 2^idx * load + 1 / 2^idx * cur_load + * + * If a cpu misses updates for n-1 ticks (as it was idle) and update gets called + * on nth tick when cpu may be busy, then we have: + * load = ((2^idx - 1) / 2^idx)^(n-1) * load + * load = (2^idx - 1) / 2^idx) * load + 1 / 2^idx * cur_load + * + * decay_load_missed() below does efficient calculation of + * load = ((2^idx - 1) / 2^idx)^(n-1) * load + * avoiding 0..n-1 loop doing load = ((2^idx - 1) / 2^idx) * load + * + * The calculation is approximated on a 128 point scale. + * degrade_zero_ticks is the number of ticks after which load at any + * particular idx is approximated to be zero. + * degrade_factor is a precomputed table, a row for each load idx. + * Each column corresponds to degradation factor for a power of two ticks, + * based on 128 point scale. + * Example: + * row 2, col 3 (=12) says that the degradation at load idx 2 after + * 8 ticks is 12/128 (which is an approximation of exact factor 3^8/4^8). + * + * With this power of 2 load factors, we can degrade the load n times + * by looking at 1 bits in n and doing as many mult/shift instead of + * n mult/shifts needed by the exact degradation. + */ +#define DEGRADE_SHIFT 7 +static const unsigned char + degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128}; +static const unsigned char + degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = { + {0, 0, 0, 0, 0, 0, 0, 0}, + {64, 32, 8, 0, 0, 0, 0, 0}, + {96, 72, 40, 12, 1, 0, 0}, + {112, 98, 75, 43, 15, 1, 0}, + {120, 112, 98, 76, 45, 16, 2} }; + +/* + * Update cpu_load for any missed ticks, due to tickless idle. The backlog + * would be when CPU is idle and so we just decay the old load without + * adding any new load. + */ +static unsigned long +decay_load_missed(unsigned long load, unsigned long missed_updates, int idx) +{ + int j = 0; + + if (!missed_updates) + return load; + + if (missed_updates >= degrade_zero_ticks[idx]) + return 0; + + if (idx == 1) + return load >> missed_updates; + + while (missed_updates) { + if (missed_updates % 2) + load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT; + + missed_updates >>= 1; + j++; + } + return load; +} + +/* + * Update rq->cpu_load[] statistics. This function is usually called every + * scheduler tick (TICK_NSEC). With tickless idle this will not be called + * every tick. We fix it up based on jiffies. + */ +static void __update_cpu_load(struct rq *this_rq, unsigned long this_load, + unsigned long pending_updates) +{ + int i, scale; + + this_rq->nr_load_updates++; + + /* Update our load: */ + this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */ + for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) { + unsigned long old_load, new_load; + + /* scale is effectively 1 << i now, and >> i divides by scale */ + + old_load = this_rq->cpu_load[i]; + old_load = decay_load_missed(old_load, pending_updates - 1, i); + new_load = this_load; + /* + * Round up the averaging division if load is increasing. This + * prevents us from getting stuck on 9 if the load is 10, for + * example. + */ + if (new_load > old_load) + new_load += scale - 1; + + this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i; + } + + sched_avg_update(this_rq); +} + +#ifdef CONFIG_NO_HZ_COMMON +/* + * There is no sane way to deal with nohz on smp when using jiffies because the + * cpu doing the jiffies update might drift wrt the cpu doing the jiffy reading + * causing off-by-one errors in observed deltas; {0,2} instead of {1,1}. + * + * Therefore we cannot use the delta approach from the regular tick since that + * would seriously skew the load calculation. However we'll make do for those + * updates happening while idle (nohz_idle_balance) or coming out of idle + * (tick_nohz_idle_exit). + * + * This means we might still be one tick off for nohz periods. + */ + +/* + * Called from nohz_idle_balance() to update the load ratings before doing the + * idle balance. + */ +static void update_idle_cpu_load(struct rq *this_rq) +{ + unsigned long curr_jiffies = ACCESS_ONCE(jiffies); + unsigned long load = this_rq->cfs.runnable_load_avg; + unsigned long pending_updates; + + /* + * bail if there's load or we're actually up-to-date. + */ + if (load || curr_jiffies == this_rq->last_load_update_tick) + return; + + pending_updates = curr_jiffies - this_rq->last_load_update_tick; + this_rq->last_load_update_tick = curr_jiffies; + + __update_cpu_load(this_rq, load, pending_updates); +} + +/* + * Called from tick_nohz_idle_exit() -- try and fix up the ticks we missed. + */ +void update_cpu_load_nohz(void) +{ + struct rq *this_rq = this_rq(); + unsigned long curr_jiffies = ACCESS_ONCE(jiffies); + unsigned long pending_updates; + + if (curr_jiffies == this_rq->last_load_update_tick) + return; + + raw_spin_lock(&this_rq->lock); + pending_updates = curr_jiffies - this_rq->last_load_update_tick; + if (pending_updates) { + this_rq->last_load_update_tick = curr_jiffies; + /* + * We were idle, this means load 0, the current load might be + * !0 due to remote wakeups and the sort. + */ + __update_cpu_load(this_rq, 0, pending_updates); + } + raw_spin_unlock(&this_rq->lock); +} +#endif /* CONFIG_NO_HZ */ + +/* + * Called from scheduler_tick() + */ +void update_cpu_load_active(struct rq *this_rq) +{ + unsigned long load = this_rq->cfs.runnable_load_avg; + /* + * See the mess around update_idle_cpu_load() / update_cpu_load_nohz(). + */ + this_rq->last_load_update_tick = jiffies; + __update_cpu_load(this_rq, load, 1); +} + /* Used instead of source_load when we know the type == 0 */ static unsigned long weighted_cpuload(const int cpu) { diff --git a/kernel/sched/proc.c b/kernel/sched/loadavg.c similarity index 62% rename from kernel/sched/proc.c rename to kernel/sched/loadavg.c index 8ecd552fe4f22..ef7159012cf36 100644 --- a/kernel/sched/proc.c +++ b/kernel/sched/loadavg.c @@ -1,7 +1,9 @@ /* - * kernel/sched/proc.c + * kernel/sched/loadavg.c * - * Kernel load calculations, forked from sched/core.c + * This file contains the magic bits required to compute the global loadavg + * figure. Its a silly number but people think its important. We go through + * great pains to make it work on big machines and tickless kernels. */ #include @@ -81,7 +83,7 @@ long calc_load_fold_active(struct rq *this_rq) long nr_active, delta = 0; nr_active = this_rq->nr_running; - nr_active += (long) this_rq->nr_uninterruptible; + nr_active += (long)this_rq->nr_uninterruptible; if (nr_active != this_rq->calc_load_active) { delta = nr_active - this_rq->calc_load_active; @@ -186,6 +188,7 @@ void calc_load_enter_idle(void) delta = calc_load_fold_active(this_rq); if (delta) { int idx = calc_load_write_idx(); + atomic_long_add(delta, &calc_load_idle[idx]); } } @@ -241,18 +244,20 @@ fixed_power_int(unsigned long x, unsigned int frac_bits, unsigned int n) { unsigned long result = 1UL << frac_bits; - if (n) for (;;) { - if (n & 1) { - result *= x; - result += 1UL << (frac_bits - 1); - result >>= frac_bits; + if (n) { + for (;;) { + if (n & 1) { + result *= x; + result += 1UL << (frac_bits - 1); + result >>= frac_bits; + } + n >>= 1; + if (!n) + break; + x *= x; + x += 1UL << (frac_bits - 1); + x >>= frac_bits; } - n >>= 1; - if (!n) - break; - x *= x; - x += 1UL << (frac_bits - 1); - x >>= frac_bits; } return result; @@ -285,7 +290,6 @@ static unsigned long calc_load_n(unsigned long load, unsigned long exp, unsigned long active, unsigned int n) { - return calc_load(load, fixed_power_int(exp, FSHIFT, n), active); } @@ -339,6 +343,8 @@ static inline void calc_global_nohz(void) { } /* * calc_load - update the avenrun load estimates 10 ticks after the * CPUs have updated calc_load_tasks. + * + * Called from the global timer code. */ void calc_global_load(unsigned long ticks) { @@ -370,10 +376,10 @@ void calc_global_load(unsigned long ticks) } /* - * Called from update_cpu_load() to periodically update this CPU's + * Called from scheduler_tick() to periodically update this CPU's * active count. */ -static void calc_load_account_active(struct rq *this_rq) +void calc_global_load_tick(struct rq *this_rq) { long delta; @@ -386,199 +392,3 @@ static void calc_load_account_active(struct rq *this_rq) this_rq->calc_load_update += LOAD_FREQ; } - -/* - * End of global load-average stuff - */ - -/* - * The exact cpuload at various idx values, calculated at every tick would be - * load = (2^idx - 1) / 2^idx * load + 1 / 2^idx * cur_load - * - * If a cpu misses updates for n-1 ticks (as it was idle) and update gets called - * on nth tick when cpu may be busy, then we have: - * load = ((2^idx - 1) / 2^idx)^(n-1) * load - * load = (2^idx - 1) / 2^idx) * load + 1 / 2^idx * cur_load - * - * decay_load_missed() below does efficient calculation of - * load = ((2^idx - 1) / 2^idx)^(n-1) * load - * avoiding 0..n-1 loop doing load = ((2^idx - 1) / 2^idx) * load - * - * The calculation is approximated on a 128 point scale. - * degrade_zero_ticks is the number of ticks after which load at any - * particular idx is approximated to be zero. - * degrade_factor is a precomputed table, a row for each load idx. - * Each column corresponds to degradation factor for a power of two ticks, - * based on 128 point scale. - * Example: - * row 2, col 3 (=12) says that the degradation at load idx 2 after - * 8 ticks is 12/128 (which is an approximation of exact factor 3^8/4^8). - * - * With this power of 2 load factors, we can degrade the load n times - * by looking at 1 bits in n and doing as many mult/shift instead of - * n mult/shifts needed by the exact degradation. - */ -#define DEGRADE_SHIFT 7 -static const unsigned char - degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128}; -static const unsigned char - degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = { - {0, 0, 0, 0, 0, 0, 0, 0}, - {64, 32, 8, 0, 0, 0, 0, 0}, - {96, 72, 40, 12, 1, 0, 0}, - {112, 98, 75, 43, 15, 1, 0}, - {120, 112, 98, 76, 45, 16, 2} }; - -/* - * Update cpu_load for any missed ticks, due to tickless idle. The backlog - * would be when CPU is idle and so we just decay the old load without - * adding any new load. - */ -static unsigned long -decay_load_missed(unsigned long load, unsigned long missed_updates, int idx) -{ - int j = 0; - - if (!missed_updates) - return load; - - if (missed_updates >= degrade_zero_ticks[idx]) - return 0; - - if (idx == 1) - return load >> missed_updates; - - while (missed_updates) { - if (missed_updates % 2) - load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT; - - missed_updates >>= 1; - j++; - } - return load; -} - -/* - * Update rq->cpu_load[] statistics. This function is usually called every - * scheduler tick (TICK_NSEC). With tickless idle this will not be called - * every tick. We fix it up based on jiffies. - */ -static void __update_cpu_load(struct rq *this_rq, unsigned long this_load, - unsigned long pending_updates) -{ - int i, scale; - - this_rq->nr_load_updates++; - - /* Update our load: */ - this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */ - for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) { - unsigned long old_load, new_load; - - /* scale is effectively 1 << i now, and >> i divides by scale */ - - old_load = this_rq->cpu_load[i]; - old_load = decay_load_missed(old_load, pending_updates - 1, i); - new_load = this_load; - /* - * Round up the averaging division if load is increasing. This - * prevents us from getting stuck on 9 if the load is 10, for - * example. - */ - if (new_load > old_load) - new_load += scale - 1; - - this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i; - } - - sched_avg_update(this_rq); -} - -#ifdef CONFIG_SMP -static inline unsigned long get_rq_runnable_load(struct rq *rq) -{ - return rq->cfs.runnable_load_avg; -} -#else -static inline unsigned long get_rq_runnable_load(struct rq *rq) -{ - return rq->load.weight; -} -#endif - -#ifdef CONFIG_NO_HZ_COMMON -/* - * There is no sane way to deal with nohz on smp when using jiffies because the - * cpu doing the jiffies update might drift wrt the cpu doing the jiffy reading - * causing off-by-one errors in observed deltas; {0,2} instead of {1,1}. - * - * Therefore we cannot use the delta approach from the regular tick since that - * would seriously skew the load calculation. However we'll make do for those - * updates happening while idle (nohz_idle_balance) or coming out of idle - * (tick_nohz_idle_exit). - * - * This means we might still be one tick off for nohz periods. - */ - -/* - * Called from nohz_idle_balance() to update the load ratings before doing the - * idle balance. - */ -void update_idle_cpu_load(struct rq *this_rq) -{ - unsigned long curr_jiffies = ACCESS_ONCE(jiffies); - unsigned long load = get_rq_runnable_load(this_rq); - unsigned long pending_updates; - - /* - * bail if there's load or we're actually up-to-date. - */ - if (load || curr_jiffies == this_rq->last_load_update_tick) - return; - - pending_updates = curr_jiffies - this_rq->last_load_update_tick; - this_rq->last_load_update_tick = curr_jiffies; - - __update_cpu_load(this_rq, load, pending_updates); -} - -/* - * Called from tick_nohz_idle_exit() -- try and fix up the ticks we missed. - */ -void update_cpu_load_nohz(void) -{ - struct rq *this_rq = this_rq(); - unsigned long curr_jiffies = ACCESS_ONCE(jiffies); - unsigned long pending_updates; - - if (curr_jiffies == this_rq->last_load_update_tick) - return; - - raw_spin_lock(&this_rq->lock); - pending_updates = curr_jiffies - this_rq->last_load_update_tick; - if (pending_updates) { - this_rq->last_load_update_tick = curr_jiffies; - /* - * We were idle, this means load 0, the current load might be - * !0 due to remote wakeups and the sort. - */ - __update_cpu_load(this_rq, 0, pending_updates); - } - raw_spin_unlock(&this_rq->lock); -} -#endif /* CONFIG_NO_HZ */ - -/* - * Called from scheduler_tick() - */ -void update_cpu_load_active(struct rq *this_rq) -{ - unsigned long load = get_rq_runnable_load(this_rq); - /* - * See the mess around update_idle_cpu_load() / update_cpu_load_nohz(). - */ - this_rq->last_load_update_tick = jiffies; - __update_cpu_load(this_rq, load, 1); - - calc_load_account_active(this_rq); -} diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index bc64c52e11712..488a56e434b97 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -25,8 +25,14 @@ extern __read_mostly int scheduler_running; extern unsigned long calc_load_update; extern atomic_long_t calc_load_tasks; +extern void calc_global_load_tick(struct rq *this_rq); extern long calc_load_fold_active(struct rq *this_rq); + +#ifdef CONFIG_SMP extern void update_cpu_load_active(struct rq *this_rq); +#else +static inline void update_cpu_load_active(struct rq *this_rq) { } +#endif /* * Helpers for converting nanosecond timing to jiffy resolution @@ -1353,8 +1359,6 @@ extern void init_dl_task_timer(struct sched_dl_entity *dl_se); unsigned long to_ratio(u64 period, u64 runtime); -extern void update_idle_cpu_load(struct rq *this_rq); - extern void init_task_runnable_average(struct task_struct *p); static inline void add_nr_running(struct rq *rq, unsigned count) From 64e0c7dbf597cb51685bea7212db79730c000ec8 Mon Sep 17 00:00:00 2001 From: Ben Segall Date: Mon, 6 Apr 2015 15:28:10 -0700 Subject: [PATCH 38/86] UPSTREAM: sched/fair: Prevent throttling in early pick_next_task_fair() The optimized task selection logic optimistically selects a new task to run without first doing a full put_prev_task(). This is so that we can avoid a put/set on the common ancestors of the old and new task. Similarly, we should only call check_cfs_rq_runtime() to throttle eligible groups if they're part of the common ancestry, otherwise it is possible to end up with no eligible task in the simple task selection. Imagine: /root /prev /next /A /B If our optimistic selection ends up throttling /next, we goto simple and our put_prev_task() ends up throttling /prev, after which we're going to bug out in set_next_entity() because there aren't any tasks left. Avoid this scenario by only throttling common ancestors. Reported-by: Mohammed Naser Reported-by: Konstantin Khlebnikov Signed-off-by: Ben Segall [ munged Changelog ] Signed-off-by: Peter Zijlstra (Intel) Cc: Andrew Morton Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Roman Gushchin Cc: Thomas Gleixner Cc: pjt@google.com Fixes: 678d5718d8d0 ("sched/fair: Optimize cgroup pick_next_task_fair()") Link: http://lkml.kernel.org/r/xm26wq1oswoq.fsf@sword-of-the-dawn.mtv.corp.google.com Signed-off-by: Ingo Molnar (cherry picked from commit 54d27365cae88fbcc853b391dcd561e71acb81fa) Signed-off-by: Javi Merino BUG=chrome-os-partner:47839 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: I0b420fc2ff8df52ed95320bdc2fcc3f0f52f17c0 Reviewed-on: https://chromium-review.googlesource.com/327011 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/fair.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 3d9708ff7f645..ce8376b73eada 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5902,18 +5902,21 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev) * entity, update_curr() will update its vruntime, otherwise * forget we've ever seen it. */ - if (curr && curr->on_rq) - update_curr(cfs_rq); - else - curr = NULL; + if (curr) { + if (curr->on_rq) + update_curr(cfs_rq); + else + curr = NULL; - /* - * This call to check_cfs_rq_runtime() will do the throttle and - * dequeue its entity in the parent(s). Therefore the 'simple' - * nr_running test will indeed be correct. - */ - if (unlikely(check_cfs_rq_runtime(cfs_rq))) - goto simple; + /* + * This call to check_cfs_rq_runtime() will do the + * throttle and dequeue its entity in the parent(s). + * Therefore the 'simple' nr_running test will indeed + * be correct. + */ + if (unlikely(check_cfs_rq_runtime(cfs_rq))) + goto simple; + } se = pick_next_entity(cfs_rq, curr); cfs_rq = group_cfs_rq(se); From 98f158b5cf07c83d354957184b16c7f90810c8c8 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 11 Jun 2015 14:46:49 +0200 Subject: [PATCH 39/86] UPSTREAM: sched,dl: Fix sched class hopping CBS hole We still have a few pending issues with the deadline code, one of which is that switching between scheduling classes can 'leak' CBS state. Close the hole by retaining the current CBS state when leaving SCHED_DEADLINE and unconditionally programming the deadline timer. The timer will then reset the CBS state if the task is still !SCHED_DEADLINE by the time it hits. If the task left SCHED_DEADLINE it will not call task_dead_dl() and we'll not cancel the hrtimer, leaving us a pending timer in free space. Avoid this by giving the timer a task reference, this avoids littering the task exit path for this rather uncommon case. In order to do this, I had to move dl_task_offline_migration() below the replenishment, such that the task_rq()->lock fully covers that. While doing this, I noticed that it (was) buggy in assuming a task is enqueued and or we need to enqueue the task now. Fixing this means select_task_rq_dl() might encounter an offline rq -- look into that. As a result this kills cancel_dl_timer() which included a rq->lock break. Fixes: 40767b0dc768 ("sched/deadline: Fix deadline parameter modification handling") Cc: Wanpeng Li Cc: Luca Abeni Cc: Juri Lelli Signed-off-by: Peter Zijlstra (Intel) Cc: ktkhai@parallels.com Cc: rostedt@goodmis.org Cc: juri.lelli@gmail.com Cc: pang.xunlei@linaro.org Cc: oleg@redhat.com Cc: wanpeng.li@linux.intel.com Cc: Luca Abeni Cc: Juri Lelli Cc: umgwanakikbuti@gmail.com Link: http://lkml.kernel.org/r/20150611124743.574192138@infradead.org Signed-off-by: Thomas Gleixner (cherry picked from commit a649f237db18450de767d70f40a41d5dbd0291de) Signed-off-by: Punit Agrawal BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: Ia764e80c68d1826ee639eb1df4db63f79d3b454b Reviewed-on: https://chromium-review.googlesource.com/321261 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/deadline.c | 152 +++++++++++++++++++++++----------------- 1 file changed, 86 insertions(+), 66 deletions(-) diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 72af5cd760393..4ea8dfcce1299 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -220,7 +220,7 @@ static inline void set_post_schedule(struct rq *rq) static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq); -static void dl_task_offline_migration(struct rq *rq, struct task_struct *p) +static struct rq *dl_task_offline_migration(struct rq *rq, struct task_struct *p) { struct rq *later_rq = NULL; bool fallback = false; @@ -254,14 +254,19 @@ static void dl_task_offline_migration(struct rq *rq, struct task_struct *p) double_lock_balance(rq, later_rq); } + /* + * By now the task is replenished and enqueued; migrate it. + */ deactivate_task(rq, p, 0); set_task_cpu(p, later_rq->cpu); - activate_task(later_rq, p, ENQUEUE_REPLENISH); + activate_task(later_rq, p, 0); if (!fallback) resched_curr(later_rq); - double_unlock_balance(rq, later_rq); + double_unlock_balance(later_rq, rq); + + return later_rq; } #else @@ -498,22 +503,23 @@ static void update_dl_entity(struct sched_dl_entity *dl_se, * actually started or not (i.e., the replenishment instant is in * the future or in the past). */ -static int start_dl_timer(struct sched_dl_entity *dl_se, bool boosted) +static int start_dl_timer(struct task_struct *p) { - struct dl_rq *dl_rq = dl_rq_of_se(dl_se); - struct rq *rq = rq_of_dl_rq(dl_rq); + struct sched_dl_entity *dl_se = &p->dl; + struct hrtimer *timer = &dl_se->dl_timer; + struct rq *rq = task_rq(p); ktime_t now, act; s64 delta; - if (boosted) - return 0; + lockdep_assert_held(&rq->lock); + /* * We want the timer to fire at the deadline, but considering * that it is actually coming from rq->clock and not from * hrtimer's time base reading. */ act = ns_to_ktime(dl_se->deadline); - now = hrtimer_cb_get_time(&dl_se->dl_timer); + now = hrtimer_cb_get_time(timer); delta = ktime_to_ns(now) - rq_clock(rq); act = ktime_add_ns(act, delta); @@ -525,7 +531,19 @@ static int start_dl_timer(struct sched_dl_entity *dl_se, bool boosted) if (ktime_us_delta(act, now) < 0) return 0; - hrtimer_start(&dl_se->dl_timer, act, HRTIMER_MODE_ABS); + /* + * !enqueued will guarantee another callback; even if one is already in + * progress. This ensures a balanced {get,put}_task_struct(). + * + * The race against __run_timer() clearing the enqueued state is + * harmless because we're holding task_rq()->lock, therefore the timer + * expiring after we've done the check will wait on its task_rq_lock() + * and observe our state. + */ + if (!hrtimer_is_queued(timer)) { + get_task_struct(p); + hrtimer_start(timer, act, HRTIMER_MODE_ABS); + } return 1; } @@ -561,35 +579,40 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) } /* - * We need to take care of several possible races here: - * - * - the task might have changed its scheduling policy - * to something different than SCHED_DEADLINE - * - the task might have changed its reservation parameters - * (through sched_setattr()) - * - the task might have been boosted by someone else and - * might be in the boosting/deboosting path + * The task might have changed its scheduling policy to something + * different than SCHED_DEADLINE (through switched_fromd_dl()). + */ + if (!dl_task(p)) { + __dl_clear_params(p); + goto unlock; + } + + /* + * This is possible if switched_from_dl() raced against a running + * callback that took the above !dl_task() path and we've since then + * switched back into SCHED_DEADLINE. * - * In all this cases we bail out, as the task is already - * in the runqueue or is going to be enqueued back anyway. + * There's nothing to do except drop our task reference. */ - if (!dl_task(p) || dl_se->dl_new || - dl_se->dl_boosted || !dl_se->dl_throttled) + if (dl_se->dl_new) goto unlock; - sched_clock_tick(); - update_rq_clock(rq); + /* + * The task might have been boosted by someone else and might be in the + * boosting/deboosting path, its not throttled. + */ + if (dl_se->dl_boosted) + goto unlock; -#ifdef CONFIG_SMP /* - * If we find that the rq the task was on is no longer - * available, we need to select a new rq. + * Spurious timer due to start_dl_timer() race; or we already received + * a replenishment from rt_mutex_setprio(). */ - if (unlikely(!rq->online)) { - dl_task_offline_migration(rq, p); + if (!dl_se->dl_throttled) goto unlock; - } -#endif + + sched_clock_tick(); + update_rq_clock(rq); /* * If the throttle happened during sched-out; like: @@ -615,17 +638,38 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) check_preempt_curr_dl(rq, p, 0); else resched_curr(rq); + #ifdef CONFIG_SMP /* - * Queueing this task back might have overloaded rq, - * check if we need to kick someone away. + * Perform balancing operations here; after the replenishments. We + * cannot drop rq->lock before this, otherwise the assertion in + * start_dl_timer() about not missing updates is not true. + * + * If we find that the rq the task was on is no longer available, we + * need to select a new rq. + * + * XXX figure out if select_task_rq_dl() deals with offline cpus. + */ + if (unlikely(!rq->online)) + rq = dl_task_offline_migration(rq, p); + + /* + * Queueing this task back might have overloaded rq, check if we need + * to kick someone away. */ if (has_pushable_dl_tasks(rq)) push_dl_task(rq); #endif + unlock: raw_spin_unlock(&rq->lock); + /* + * This can free the task_struct, including this hrtimer, do not touch + * anything related to that after this. + */ + put_task_struct(p); + return HRTIMER_NORESTART; } @@ -685,7 +729,7 @@ static void update_curr_dl(struct rq *rq) if (dl_runtime_exceeded(rq, dl_se)) { dl_se->dl_throttled = 1; __dequeue_task_dl(rq, curr, 0); - if (unlikely(!start_dl_timer(dl_se, curr->dl.dl_boosted))) + if (unlikely(dl_se->dl_boosted || !start_dl_timer(curr))) enqueue_task_dl(rq, curr, ENQUEUE_REPLENISH); if (!is_leftmost(curr, &rq->dl)) @@ -1156,7 +1200,6 @@ static void task_fork_dl(struct task_struct *p) static void task_dead_dl(struct task_struct *p) { - struct hrtimer *timer = &p->dl.dl_timer; struct dl_bw *dl_b = dl_bw_of(task_cpu(p)); /* @@ -1166,8 +1209,6 @@ static void task_dead_dl(struct task_struct *p) /* XXX we should retain the bw until 0-lag */ dl_b->total_bw -= p->dl.dl_bw; raw_spin_unlock_irq(&dl_b->lock); - - hrtimer_cancel(timer); } static void set_curr_task_dl(struct rq *rq) @@ -1634,37 +1675,16 @@ void init_sched_dl_class(void) #endif /* CONFIG_SMP */ -/* - * Ensure p's dl_timer is cancelled. May drop rq->lock for a while. - */ -static void cancel_dl_timer(struct rq *rq, struct task_struct *p) -{ - struct hrtimer *dl_timer = &p->dl.dl_timer; - - /* Nobody will change task's class if pi_lock is held */ - lockdep_assert_held(&p->pi_lock); - - if (hrtimer_active(dl_timer)) { - int ret = hrtimer_try_to_cancel(dl_timer); - - if (unlikely(ret == -1)) { - /* - * Note, p may migrate OR new deadline tasks - * may appear in rq when we are unlocking it. - * A caller of us must be fine with that. - */ - raw_spin_unlock(&rq->lock); - hrtimer_cancel(dl_timer); - raw_spin_lock(&rq->lock); - } - } -} - static void switched_from_dl(struct rq *rq, struct task_struct *p) { - /* XXX we should retain the bw until 0-lag */ - cancel_dl_timer(rq, p); - __dl_clear_params(p); + /* + * Start the deadline timer; if we switch back to dl before this we'll + * continue consuming our current CBS slice. If we stay outside of + * SCHED_DEADLINE until the deadline passes, the timer will reset the + * task. + */ + if (!start_dl_timer(p)) + __dl_clear_params(p); /* * Since this might be the only -deadline task on the rq, From 0ae89cfbdb2597a14fdf88151c43b6c020ab8b50 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 11 Jun 2015 14:46:50 +0200 Subject: [PATCH 40/86] UPSTREAM: sched: Move code around In preparation to reworking set_cpus_allowed_ptr() move some code around. This also removes some superfluous #ifdefs and adds comments to some #endifs. text data bss dec hex filename 12211532 1738144 1081344 15031020 e55aec defconfig-build/vmlinux.pre 12211532 1738144 1081344 15031020 e55aec defconfig-build/vmlinux.post Signed-off-by: Peter Zijlstra (Intel) Cc: ktkhai@parallels.com Cc: rostedt@goodmis.org Cc: juri.lelli@gmail.com Cc: pang.xunlei@linaro.org Cc: oleg@redhat.com Cc: wanpeng.li@linux.intel.com Cc: umgwanakikbuti@gmail.com Link: http://lkml.kernel.org/r/20150611124743.662086684@infradead.org Signed-off-by: Thomas Gleixner (cherry picked from commit 5cc389bcee088b72c8c34a01d596412cab4f3f78) Signed-off-by: Ricky Liang BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: I856df8f67caa2300a6713ebaced9974a0e2db0bf Reviewed-on: https://chromium-review.googlesource.com/321262 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/core.c | 364 ++++++++++++++++++++++---------------------- 1 file changed, 178 insertions(+), 186 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 25fb2b975a3e2..0df50556e944a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1055,6 +1055,180 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) } #ifdef CONFIG_SMP +/* + * This is how migration works: + * + * 1) we invoke migration_cpu_stop() on the target CPU using + * stop_one_cpu(). + * 2) stopper starts to run (implicitly forcing the migrated thread + * off the CPU) + * 3) it checks whether the migrated task is still in the wrong runqueue. + * 4) if it's in the wrong runqueue then the migration thread removes + * it and puts it into the right queue. + * 5) stopper completes and stop_one_cpu() returns and the migration + * is done. + */ + +/* + * move_queued_task - move a queued task to new rq. + * + * Returns (locked) new rq. Old rq's lock is released. + */ +static struct rq *move_queued_task(struct task_struct *p, int new_cpu) +{ + struct rq *rq = task_rq(p); + + lockdep_assert_held(&rq->lock); + + dequeue_task(rq, p, 0); + p->on_rq = TASK_ON_RQ_MIGRATING; + set_task_cpu(p, new_cpu); + raw_spin_unlock(&rq->lock); + + rq = cpu_rq(new_cpu); + + raw_spin_lock(&rq->lock); + BUG_ON(task_cpu(p) != new_cpu); + p->on_rq = TASK_ON_RQ_QUEUED; + enqueue_task(rq, p, 0); + check_preempt_curr(rq, p, 0); + + return rq; +} + +struct migration_arg { + struct task_struct *task; + int dest_cpu; +}; + +/* + * Move (not current) task off this cpu, onto dest cpu. We're doing + * this because either it can't run here any more (set_cpus_allowed() + * away from this CPU, or CPU going down), or because we're + * attempting to rebalance this task on exec (sched_exec). + * + * So we race with normal scheduler movements, but that's OK, as long + * as the task is no longer on this CPU. + * + * Returns non-zero if task was successfully migrated. + */ +static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) +{ + struct rq *rq; + int ret = 0; + + if (unlikely(!cpu_active(dest_cpu))) + return ret; + + rq = cpu_rq(src_cpu); + + raw_spin_lock(&p->pi_lock); + raw_spin_lock(&rq->lock); + /* Already moved. */ + if (task_cpu(p) != src_cpu) + goto done; + + /* Affinity changed (again). */ + if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) + goto fail; + + /* + * If we're not on a rq, the next wake-up will ensure we're + * placed properly. + */ + if (task_on_rq_queued(p)) + rq = move_queued_task(p, dest_cpu); +done: + ret = 1; +fail: + raw_spin_unlock(&rq->lock); + raw_spin_unlock(&p->pi_lock); + return ret; +} + +/* + * migration_cpu_stop - this will be executed by a highprio stopper thread + * and performs thread migration by bumping thread off CPU then + * 'pushing' onto another runqueue. + */ +static int migration_cpu_stop(void *data) +{ + struct migration_arg *arg = data; + + /* + * The original target cpu might have gone down and we might + * be on another cpu but it doesn't matter. + */ + local_irq_disable(); + /* + * We need to explicitly wake pending tasks before running + * __migrate_task() such that we will not miss enforcing cpus_allowed + * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. + */ + sched_ttwu_pending(); + __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu); + local_irq_enable(); + return 0; +} + +void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) +{ + if (p->sched_class->set_cpus_allowed) + p->sched_class->set_cpus_allowed(p, new_mask); + + cpumask_copy(&p->cpus_allowed, new_mask); + p->nr_cpus_allowed = cpumask_weight(new_mask); +} + +/* + * Change a given task's CPU affinity. Migrate the thread to a + * proper CPU and schedule it away if the CPU it's executing on + * is removed from the allowed bitmask. + * + * NOTE: the caller must have a valid reference to the task, the + * task must not exit() & deallocate itself prematurely. The + * call is not atomic; no spinlocks may be held. + */ +int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) +{ + unsigned long flags; + struct rq *rq; + unsigned int dest_cpu; + int ret = 0; + + rq = task_rq_lock(p, &flags); + + if (cpumask_equal(&p->cpus_allowed, new_mask)) + goto out; + + if (!cpumask_intersects(new_mask, cpu_active_mask)) { + ret = -EINVAL; + goto out; + } + + do_set_cpus_allowed(p, new_mask); + + /* Can the task run on the task's current CPU? If so, we're done */ + if (cpumask_test_cpu(task_cpu(p), new_mask)) + goto out; + + dest_cpu = cpumask_any_and(cpu_active_mask, new_mask); + if (task_running(rq, p) || p->state == TASK_WAKING) { + struct migration_arg arg = { p, dest_cpu }; + /* Need help from migration thread: drop lock and wait. */ + task_rq_unlock(rq, p, &flags); + stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); + tlb_migrate_finish(p->mm); + return 0; + } else if (task_on_rq_queued(p)) + rq = move_queued_task(p, dest_cpu); +out: + task_rq_unlock(rq, p, &flags); + + return ret; +} +EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); + void set_task_cpu(struct task_struct *p, unsigned int new_cpu) { #ifdef CONFIG_SCHED_DEBUG @@ -1203,13 +1377,6 @@ int migrate_swap(struct task_struct *cur, struct task_struct *p) return ret; } -struct migration_arg { - struct task_struct *task; - int dest_cpu; -}; - -static int migration_cpu_stop(void *data); - /* * wait_task_inactive - wait for a thread to unschedule. * @@ -1342,9 +1509,7 @@ void kick_process(struct task_struct *p) preempt_enable(); } EXPORT_SYMBOL_GPL(kick_process); -#endif /* CONFIG_SMP */ -#ifdef CONFIG_SMP /* * ->cpus_allowed is protected by both rq->lock and p->pi_lock */ @@ -1448,7 +1613,7 @@ static void update_avg(u64 *avg, u64 sample) s64 diff = sample - *avg; *avg += diff >> 3; } -#endif +#endif /* CONFIG_SMP */ static void ttwu_stat(struct task_struct *p, int cpu, int wake_flags) @@ -4796,149 +4961,6 @@ void init_idle(struct task_struct *idle, int cpu) } #ifdef CONFIG_SMP -/* - * move_queued_task - move a queued task to new rq. - * - * Returns (locked) new rq. Old rq's lock is released. - */ -static struct rq *move_queued_task(struct task_struct *p, int new_cpu) -{ - struct rq *rq = task_rq(p); - - lockdep_assert_held(&rq->lock); - - dequeue_task(rq, p, 0); - p->on_rq = TASK_ON_RQ_MIGRATING; - set_task_cpu(p, new_cpu); - raw_spin_unlock(&rq->lock); - - rq = cpu_rq(new_cpu); - - raw_spin_lock(&rq->lock); - BUG_ON(task_cpu(p) != new_cpu); - p->on_rq = TASK_ON_RQ_QUEUED; - enqueue_task(rq, p, 0); - check_preempt_curr(rq, p, 0); - - return rq; -} - -void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) -{ - if (p->sched_class->set_cpus_allowed) - p->sched_class->set_cpus_allowed(p, new_mask); - - cpumask_copy(&p->cpus_allowed, new_mask); - p->nr_cpus_allowed = cpumask_weight(new_mask); -} - -/* - * This is how migration works: - * - * 1) we invoke migration_cpu_stop() on the target CPU using - * stop_one_cpu(). - * 2) stopper starts to run (implicitly forcing the migrated thread - * off the CPU) - * 3) it checks whether the migrated task is still in the wrong runqueue. - * 4) if it's in the wrong runqueue then the migration thread removes - * it and puts it into the right queue. - * 5) stopper completes and stop_one_cpu() returns and the migration - * is done. - */ - -/* - * Change a given task's CPU affinity. Migrate the thread to a - * proper CPU and schedule it away if the CPU it's executing on - * is removed from the allowed bitmask. - * - * NOTE: the caller must have a valid reference to the task, the - * task must not exit() & deallocate itself prematurely. The - * call is not atomic; no spinlocks may be held. - */ -int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) -{ - unsigned long flags; - struct rq *rq; - unsigned int dest_cpu; - int ret = 0; - - rq = task_rq_lock(p, &flags); - - if (cpumask_equal(&p->cpus_allowed, new_mask)) - goto out; - - if (!cpumask_intersects(new_mask, cpu_active_mask)) { - ret = -EINVAL; - goto out; - } - - do_set_cpus_allowed(p, new_mask); - - /* Can the task run on the task's current CPU? If so, we're done */ - if (cpumask_test_cpu(task_cpu(p), new_mask)) - goto out; - - dest_cpu = cpumask_any_and(cpu_active_mask, new_mask); - if (task_running(rq, p) || p->state == TASK_WAKING) { - struct migration_arg arg = { p, dest_cpu }; - /* Need help from migration thread: drop lock and wait. */ - task_rq_unlock(rq, p, &flags); - stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); - tlb_migrate_finish(p->mm); - return 0; - } else if (task_on_rq_queued(p)) - rq = move_queued_task(p, dest_cpu); -out: - task_rq_unlock(rq, p, &flags); - - return ret; -} -EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); - -/* - * Move (not current) task off this cpu, onto dest cpu. We're doing - * this because either it can't run here any more (set_cpus_allowed() - * away from this CPU, or CPU going down), or because we're - * attempting to rebalance this task on exec (sched_exec). - * - * So we race with normal scheduler movements, but that's OK, as long - * as the task is no longer on this CPU. - * - * Returns non-zero if task was successfully migrated. - */ -static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) -{ - struct rq *rq; - int ret = 0; - - if (unlikely(!cpu_active(dest_cpu))) - return ret; - - rq = cpu_rq(src_cpu); - - raw_spin_lock(&p->pi_lock); - raw_spin_lock(&rq->lock); - /* Already moved. */ - if (task_cpu(p) != src_cpu) - goto done; - - /* Affinity changed (again). */ - if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) - goto fail; - - /* - * If we're not on a rq, the next wake-up will ensure we're - * placed properly. - */ - if (task_on_rq_queued(p)) - rq = move_queued_task(p, dest_cpu); -done: - ret = 1; -fail: - raw_spin_unlock(&rq->lock); - raw_spin_unlock(&p->pi_lock); - return ret; -} #ifdef CONFIG_NUMA_BALANCING /* Migrate current task p to target_cpu */ @@ -4986,35 +5008,9 @@ void sched_setnuma(struct task_struct *p, int nid) enqueue_task(rq, p, 0); task_rq_unlock(rq, p, &flags); } -#endif - -/* - * migration_cpu_stop - this will be executed by a highprio stopper thread - * and performs thread migration by bumping thread off CPU then - * 'pushing' onto another runqueue. - */ -static int migration_cpu_stop(void *data) -{ - struct migration_arg *arg = data; - - /* - * The original target cpu might have gone down and we might - * be on another cpu but it doesn't matter. - */ - local_irq_disable(); - /* - * We need to explicitly wake pending tasks before running - * __migrate_task() such that we will not miss enforcing cpus_allowed - * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. - */ - sched_ttwu_pending(); - __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu); - local_irq_enable(); - return 0; -} +#endif /* CONFIG_NUMA_BALANCING */ #ifdef CONFIG_HOTPLUG_CPU - /* * Ensures that the idle task is using init_mm right before its cpu goes * offline. @@ -5117,7 +5113,6 @@ static void migrate_tasks(unsigned int dead_cpu) rq->stop = stop; } - #endif /* CONFIG_HOTPLUG_CPU */ #if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL) @@ -5361,7 +5356,7 @@ static void register_sched_domain_sysctl(void) static void unregister_sched_domain_sysctl(void) { } -#endif +#endif /* CONFIG_SCHED_DEBUG && CONFIG_SYSCTL */ static void set_rq_online(struct rq *rq) { @@ -5531,9 +5526,6 @@ static int __init migration_init(void) return 0; } early_initcall(migration_init); -#endif - -#ifdef CONFIG_SMP static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */ @@ -6806,7 +6798,7 @@ static int __sdt_alloc(const struct cpumask *cpu_map) BUG_ON(!nr_idle_states || !nr_cap_states); } - sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(), + sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(), GFP_KERNEL, cpu_to_node(j)); if (!sd) return -ENOMEM; From f11da02b845341529cbb1c1f0b2479f399382744 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Thu, 2 Jul 2015 22:25:52 +0800 Subject: [PATCH 41/86] UPSTREAM: sched/fair: Clean up the __sched_period() code Since commit: 4bf0b77158 ("sched: remove do_div() from __sched_slice()") ... the logic of __sched_period() can be implemented as a single if-else without any local variables, so this patch cleans it up with an if-else statement, which expresses the function's logic straightforwardly. Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Srikar Dronamraju Cc: Thomas Gleixner Link: http://lkml.kernel.org/r/1435847152-29543-1-git-send-email-boqun.feng@gmail.com Signed-off-by: Ingo Molnar (cherry picked from commit 8e2b0bf397279878babcb39b021edcafe7c945eb) Signed-off-by: Javi Merino BUG=chrome-os-partner:47839 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: I3cfee2abcbb8674b2a192168cbca96d9b9655ffa Reviewed-on: https://chromium-review.googlesource.com/327010 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/fair.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ce8376b73eada..ed45503869c43 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -637,15 +637,10 @@ static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) */ static u64 __sched_period(unsigned long nr_running) { - u64 period = sysctl_sched_latency; - unsigned long nr_latency = sched_nr_latency; - - if (unlikely(nr_running > nr_latency)) { - period = sysctl_sched_min_granularity; - period *= nr_running; - } - - return period; + if (unlikely(nr_running > sched_nr_latency)) + return nr_running * sysctl_sched_min_granularity; + else + return sysctl_sched_latency; } /* From c6e8cf24c9e08802a154963d3fa47a6c9a7e9d10 Mon Sep 17 00:00:00 2001 From: Byungchul Park Date: Mon, 10 Aug 2015 18:02:55 +0900 Subject: [PATCH 42/86] UPSTREAM: sched: Ensure a task has a non-normalized vruntime when returning back to CFS Current code ensures that a task has a normalized vruntime when switching away from the fair class, but it does not ensure the task has a non-normalized vruntime when switching back to the fair class. This is an example breaking this consistency: 1. a task is in fair class and !queued 2. changes its class to RT class (still !queued) 3. changes its class to fair class again (still !queued) BUG=chrome-os-partner:47839 TEST=Boot kernel on Oak. TEST=smaug-release and strago-release trybots. Change-Id: Iaa6c7d9e7b60551ce005ed4b5abbdbcea768a58f Reviewed-on: https://chromium-review.googlesource.com/327013 Commit-Ready: Ricky Liang Tested-by: Ricky Liang Reviewed-by: Ricky Liang --- kernel/sched/fair.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ed45503869c43..3324eaee19fb5 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8889,16 +8889,31 @@ static void switched_from_fair(struct rq *rq, struct task_struct *p) */ static void switched_to_fair(struct rq *rq, struct task_struct *p) { -#ifdef CONFIG_FAIR_GROUP_SCHED struct sched_entity *se = &p->se; + +#ifdef CONFIG_FAIR_GROUP_SCHED /* * Since the real-depth could have been changed (only FAIR * class maintain depth value), reset depth properly. */ se->depth = se->parent ? se->parent->depth + 1 : 0; #endif - if (!task_on_rq_queued(p)) + + if (!task_on_rq_queued(p)) { + + /* + * Ensure the task has a non-normalized vruntime when it is switched + * back to the fair class with !queued, so that enqueue_entity() at + * wake-up time will do the right thing. + * + * If it's queued, then the enqueue_entity(.flags=0) makes the task + * has non-normalized vruntime, if it's !queued, then it still has + * normalized vruntime. + */ + if (p->state != TASK_RUNNING) + se->vruntime += cfs_rq_of(se)->min_vruntime; return; + } /* * We were most likely switched from sched_rt, so From 66dddbfc9f4d0a1a22c21a7f79317cfc8e57503f Mon Sep 17 00:00:00 2001 From: Morten Rasmussen Date: Tue, 28 Jul 2015 15:42:47 +0100 Subject: [PATCH 43/86] CHROMIUM: sched: Consider misfit tasks when load-balancing With the new group_misfit_task load-balancing scenario additional policy conditions are needed when load-balancing. Misfit task balancing only makes sense between source group with lower capacity than the target group. If capacities are the same, fallback to normal group_other balancing. The aim is to balance tasks such that no task has its throughput hindered by compute capacity if a cpu with more capacity is available. Load-balancing is generally based on average load in the sched_groups, but for misfitting tasks it is necessary to introduce exceptions to migrate tasks against usual metrics and optimize throughput. This patch ensures the following load-balance for mixed capacity systems (e.g. ARM big.LITTLE) for always-running tasks: 1. Place a task on each cpu starting in order from cpus with highest capacity to lowest until all cpus are in use (i.e. one task on each cpu). 2. Once all cpus are in use balance according to compute capacity such that load per capacity is approximately the same regardless of the compute capacity (i.e. big cpus get more tasks than little cpus). Necessary changes are introduced in find_busiest_group(), calculate_imbalance(), and find_busiest_queue(). This includes passing the group_type on to find_busiest_queue() through struct lb_env, which is currently only considers imbalance and not the imbalance situation (group_type). To avoid taking remote rq locks to examine source sched_groups for misfit tasks, each cpu is responsible for tracking misfit tasks themselves and update the rq->misfit_task flag. This means checking task utilization when tasks are scheduled and on sched_tick. BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. Change-Id: I458461cebf269d6d4eeac6f83e4c84f4e4d7a9dd Signed-off-by: Morten Rasmussen Signed-off-by: Ricky Liang --- kernel/sched/fair.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 3324eaee19fb5..74f4e8ee64ce2 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7252,6 +7252,15 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd sgs->group_type = group_overloaded; } + /* + * Ignore task groups with misfit tasks if local group has no + * capacity or if per-cpu capacity isn't higher. + */ + if (sgs->group_type == group_misfit_task && + (!group_has_capacity(env, &sds->local_stat) || + !group_smaller_cpu_capacity(sg, sds->local))) + sgs->group_type = group_other; + if (update_sd_pick_busiest(env, sds, sg, sgs)) { sds->busiest = sg; sds->busiest_stat = *sgs; From 9d51ce10eb44decb6c6a46976c329c2e1ac66c50 Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Fri, 14 Nov 2014 16:25:50 +0000 Subject: [PATCH 44/86] DEBUG: sched: add energy procfs interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch makes the energy data available via procfs. The related files are placed as sub-directory named 'energy' inside the /proc/sys/kernel/sched_domain/cpuX/domainY/groupZ directory for those cpu/domain/group tuples which have energy information. The following example depicts the contents of /proc/sys/kernel/sched_domain/cpu0/domain0/group[01] for a system which has energy information attached to domain level 0. ├── cpu0 │ ├── domain0 │ │ ├── busy_factor │ │ ├── busy_idx │ │ ├── cache_nice_tries │ │ ├── flags │ │ ├── forkexec_idx │ │ ├── group0 │ │ │ └── energy │ │ │ ├── cap_states │ │ │ ├── idle_states │ │ │ ├── nr_cap_states │ │ │ └── nr_idle_states │ │ ├── group1 │ │ │ └── energy │ │ │ ├── cap_states │ │ │ ├── idle_states │ │ │ ├── nr_cap_states │ │ │ └── nr_idle_states │ │ ├── idle_idx │ │ ├── imbalance_pct │ │ ├── max_interval │ │ ├── max_newidle_lb_cost │ │ ├── min_interval │ │ ├── name │ │ ├── newidle_idx │ │ └── wake_idx │ └── domain1 │ ├── busy_factor │ ├── busy_idx │ ├── cache_nice_tries │ ├── flags │ ├── forkexec_idx │ ├── idle_idx │ ├── imbalance_pct │ ├── max_interval │ ├── max_newidle_lb_cost │ ├── min_interval │ ├── name │ ├── newidle_idx │ └── wake_idx The files 'nr_idle_states' and 'nr_cap_states' contain a scalar value whereas 'idle_states' and 'cap_states' contain a vector of power consumption at this idle state respectively (compute capacity, power consumption) at this capacity state. BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. Change-Id: I7457d4073b4797336a30711407220bd2eb111396 Signed-off-by: Dietmar Eggemann --- kernel/sched/core.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0df50556e944a..493ded52b5581 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5187,7 +5187,7 @@ set_table_entry(struct ctl_table *entry, static struct ctl_table * sd_alloc_ctl_energy_table(struct sched_group_energy *sge) { - struct ctl_table *table = sd_alloc_ctl_entry(6); + struct ctl_table *table = sd_alloc_ctl_entry(5); if (table == NULL) return NULL; @@ -5197,11 +5197,9 @@ sd_alloc_ctl_energy_table(struct sched_group_energy *sge) set_table_entry(&table[1], "idle_states", &sge->idle_states[0].power, sge->nr_idle_states*sizeof(struct idle_state), 0644, proc_doulongvec_minmax, false); - set_table_entry(&table[2], "nr_idle_states_below", &sge->nr_idle_states_below, + set_table_entry(&table[2], "nr_cap_states", &sge->nr_cap_states, sizeof(int), 0644, proc_dointvec_minmax, false); - set_table_entry(&table[3], "nr_cap_states", &sge->nr_cap_states, - sizeof(int), 0644, proc_dointvec_minmax, false); - set_table_entry(&table[4], "cap_states", &sge->cap_states[0].cap, + set_table_entry(&table[3], "cap_states", &sge->cap_states[0].cap, sge->nr_cap_states*sizeof(struct capacity_state), 0644, proc_doulongvec_minmax, false); @@ -5218,7 +5216,7 @@ sd_alloc_ctl_group_table(struct sched_group *sg) table->procname = kstrdup("energy", GFP_KERNEL); table->mode = 0555; - table->child = sd_alloc_ctl_energy_table(sg->sge); + table->child = sd_alloc_ctl_energy_table((struct sched_group_energy *)sg->sge); return table; } From b9a59f0b109cb826d57c36b42bef2637085cca0a Mon Sep 17 00:00:00 2001 From: Morten Rasmussen Date: Tue, 3 Feb 2015 13:54:11 +0000 Subject: [PATCH 45/86] CHROMIUM: sched: Disable energy-unfriendly nohz kicks With energy-aware scheduling enabled nohz_kick_needed() generates many nohz idle-balance kicks which lead to nothing when multiple tasks get packed on a single cpu to save energy. This causes unnecessary wake-ups and hence wastes energy. Make these conditions depend on !energy_aware() for now until the energy-aware nohz story gets sorted out. BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. cc: Ingo Molnar cc: Peter Zijlstra Signed-off-by: Morten Rasmussen Change-Id: Ieda9ec81c01795f8245c0dd3eb39ea42dba0fa32 Signed-off-by: Ricky Liang From 3aa9da8741825610231ddc88de0e989569c572ab Mon Sep 17 00:00:00 2001 From: Patrick Bellasi Date: Tue, 30 Jun 2015 12:03:26 +0100 Subject: [PATCH 46/86] WIP: sched/tune: add detailed documentation The topic of a single simple power-performance tunable, that is wholly scheduler centric, and has well defined and predictable properties has come up on several occasions in the past. With techniques such as a scheduler driven DVFS, we now have a good framework for implementing such a tunable. This patch provides a detailed description of the motivations and design decisions behind the implementation of the SchedTune. BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. Change-Id: Ia3c269ecdaab80f1ea8f79a34b795a5c7504a881 cc: Jonathan Corbet cc: linux-doc@vger.kernel.org Signed-off-by: Patrick Bellasi --- Documentation/scheduler/sched-tune.txt | 367 +++++++++++++++++++++++++ 1 file changed, 367 insertions(+) create mode 100644 Documentation/scheduler/sched-tune.txt diff --git a/Documentation/scheduler/sched-tune.txt b/Documentation/scheduler/sched-tune.txt new file mode 100644 index 0000000000000..cb795e643eba1 --- /dev/null +++ b/Documentation/scheduler/sched-tune.txt @@ -0,0 +1,367 @@ + Central, scheduler-driven, power-performance control + (EXPERIMENTAL) + +Abstract +======== + +The topic of a single simple power-performance tunable, that is wholly +scheduler centric, and has well defined and predictable properties has come up +on several occasions in the past [1,2]. With techniques such as a scheduler +driven DVFS [3], we now have a good framework for implementing such a tunable. +This document describes the overall ideas behind its design and implementation. + + +Table of Contents +================= + +1. Motivation +2. Introduction +3. Signal Boosting Strategy +4. OPP selection using boosted CPU utilization +5. Per task group boosting +6. Question and Answers + - What about "auto" mode? + - What about boosting on a congested system? + - How CPUs are boosted when we have tasks with multiple boost values? +7. References + + +1. Motivation +============= + +Sched-DVFS [3] is a new event-driven cpufreq governor which allows the +scheduler to select the optimal DVFS operating point (OPP) for running a task +allocated to a CPU. The introduction of sched-DVFS enables running workloads at +the most energy efficient OPPs. + +However, sometimes it may be desired to intentionally boost the performance of +a workload even if that could imply a reasonable increase in energy +consumption. For example, in order to reduce the response time of a task, we +may want to run the task at a higher OPP than the one that is actually required +by it's CPU bandwidth demand. + +This last requirement is especially important if we consider that one of the +main goals of the sched-DVFS component is to replace all currently available +CPUFreq policies. Since sched-DVFS is event based, as opposed to the sampling +driven governors we currently have, it is already more responsive at selecting +the optimal OPP to run tasks allocated to a CPU. However, just tracking the +actual task load demand may not be enough from a performance standpoint. For +example, it is not possible to get behaviors similar to those provided by the +"performance" and "interactive" CPUFreq governors. + +This document describes an implementation of a tunable, stacked on top of the +sched-DVFS which extends its functionality to support task performance +boosting. + +By "performance boosting" we mean the reduction of the time required to +complete a task activation, i.e. the time elapsed from a task wakeup to its +next deactivation (e.g. because it goes back to sleep or it terminates). For +example, if we consider a simple periodic task which executes the same workload +for 5[s] every 20[s] while running at a certain OPP, a boosted execution of +that task must complete each of its activations in less than 5[s]. + +A previous attempt [5] to introduce such a boosting feature has not been +successful mainly because of the complexity of the proposed solution. The +approach described in this document exposes a single simple interface to +user-space. This single tunable knob allows the tuning of system wide +scheduler behaviours ranging from energy efficiency at one end through to +incremental performance boosting at the other end. This first tunable affects +all tasks. However, a more advanced extension of the concept is also provided +which uses CGroups to boost the performance of only selected tasks while using +the energy efficient default for all others. + +The rest of this document introduces in more details the proposed solution +which has been named SchedTune. + + +2. Introduction +=============== + +SchedTune exposes a simple user-space interface with a single power-performance +tunable: + + /proc/sys/kernel/sched_cfs_boost + +This permits expressing a boost value as an integer in the range [0..100]. + +A value of 0 (default) configures the CFS scheduler for maximum energy +efficiency. This means that sched-DVFS runs the tasks at the minimum OPP +required to satisfy their workload demand. +A value of 100 configures scheduler for maximum performance, which translates +to the selection of the maximum OPP on that CPU. + +The range between 0 and 100 can be set to satisfy other scenarios suitably. For +example to satisfy interactive response or depending on other system events +(battery level etc). + +A CGroup based extension is also provided, which permits further user-space +defined task classification to tune the scheduler for different goals depending +on the specific nature of the task, e.g. background vs interactive vs +low-priority. + +The overall design of the SchedTune module is built on top of "Per-Entity Load +Tracking" (PELT) signals and sched-DVFS by introducing a bias on the Operating +Performance Point (OPP) selection. +Each time a task is allocated on a CPU, sched-DVFS has the opportunity to tune +the operating frequency of that CPU to better match the workload demand. The +selection of the actual OPP being activated is influenced by the global boost +value, or the boost value for the task CGroup when in use. + +This simple biasing approach leverages existing frameworks, which means minimal +modifications to the scheduler, and yet it allows to achieve a range of +different behaviours all from a single simple tunable knob. +The only new concept introduced is that of signal boosting. + + +3. Signal Boosting Strategy +=========================== + +The whole PELT machinery works based on the value of a few load tracking signals +which basically track the CPU bandwidth requirements for tasks and the capacity +of CPUs. The basic idea behind the SchedTune knob is to artificially inflate +some of these load tracking signals to make a task or RQ appears more demanding +that it actually is. + +Which signals have to be inflated depends on the specific "consumer". However, +independently from the specific (signal, consumer) pair, it is important to +define a simple and possibly consistent strategy for the concept of boosting a +signal. + +A boosting strategy defines how the "abstract" user-space defined +sched_cfs_boost value is translated into an internal "margin" value to be added +to a signal to get its inflated value: + + margin := boosting_strategy(sched_cfs_boost, signal) + boosted_signal := signal + margin + +Different boosting strategies were identified and analyzed before selecting the +one found to be most effective. + +Signal Proportional Compensation (SPC) +-------------------------------------- + +In this boosting strategy the sched_cfs_boost value is used to compute a +margin which is proportional to the complement of the original signal. +When a signal has a maximum possible value, its complement is defined as +the delta from the actual value and its possible maximum. + +Since the tunable implementation uses signals which have SCHED_LOAD_SCALE as +the maximum possible value, the margin becomes: + + margin := sched_cfs_boost * (SCHED_LOAD_SCALE - signal) + +Using this boosting strategy: +- a 100% sched_cfs_boost means that the signal is scaled to the maximum value +- each value in the range of sched_cfs_boost effectively inflates the signal in + question by a quantity which is proportional to the maximum value. + +For example, by applying the SPC boosting strategy to the selection of the OPP +to run a task it is possible to achieve these behaviors: + +- 0% boosting: run the task at the minimum OPP required by its workload +- 100% boosting: run the task at the maximum OPP available for the CPU +- 50% boosting: run at the half-way OPP between minimum and maximum + +Which means that, at 50% boosting, a task will be scheduled to run at half of +the maximum theoretically achievable performance on the specific target +platform. + +A graphical representation of an SPC boosted signal is represented in the +following figure where: + a) "-" represents the original signal + b) "b" represents a 50% boosted signal + c) "p" represents a 100% boosted signal + + + ^ + | SCHED_LOAD_SCALE + +-----------------------------------------------------------------+ + |pppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp + | + | boosted_signal + | bbbbbbbbbbbbbbbbbbbbbbbb + | + | original signal + | bbbbbbbbbbbbbbbbbbbbbbbb+----------------------+ + | | + |bbbbbbbbbbbbbbbbbb | + | | + | | + | | + | +-----------------------+ + | | + | | + | | + |------------------+ + | + | + +-----------------------------------------------------------------------> + +The plot above shows a ramped load signal (titled 'original_signal') and it's +boosted equivalent. For each step of the original signal the boosted signal +corresponding to a 50% boost is midway from the original signal and the upper +bound. Boosting by 100% generates a boosted signal which is always saturated to +the upper bound. + + +4. OPP selection using boosted CPU utilization +============================================== + +It is worth calling out that the implementation does not introduce any new load +signals. Instead, it provides an API to tune existing signals. This tuning is +done on demand and only in scheduler code paths where it is sensible to do so. +The new API calls are defined to return either the default signal or a boosted +one, depending on the value of sched_cfs_boost. This is a clean an non invasive +modification of the existing existing code paths. + +The signal representing a CPU's utilization is boosted according to the +previously described SPC boosting strategy. To sched-DVFS, this allows a CPU +(ie CFS run-queue) to appear more used then it actually is. + +Thus, with the sched_cfs_boost enabled we have the following main functions to +get the current utilization of a CPU: + + cpu_util() + boosted_cpu_util() + +The new boosted_cpu_util() is similar to the first but returns a boosted +utilization signal which is a function of the sched_cfs_boost value. + +This function is used in the CFS scheduler code paths where sched-DVFS needs to +decide the OPP to run a CPU at. +For example, this allows selecting the highest OPP for a CPU which has +the boost value set to 100%. + + +5. Per task group boosting +========================== + +The availability of a single knob which is used to boost all tasks in the +system is certainly a simple solution but it quite likely doesn't fit many +utilization scenarios, especially in the mobile device space. + +For example, on battery powered devices there usually are many background +services which are long running and need energy efficient scheduling. On the +other hand, some applications are more performance sensitive and require an +interactive response and/or maximum performance, regardless of the energy cost. +To better service such scenarios, the SchedTune implementation has an extension +that provides a more fine grained boosting interface. + +A new CGroup controller, namely "schedtune", could be enabled which allows to +defined and configure task groups with different boosting values. +Tasks that require special performance can be put into separate CGroups. +The value of the boost associated with the tasks in this group can be specified +using a single knob exposed by the CGroup controller: + + schedtune.boost + +This knob allows the definition of a boost value that is to be used for +SPC boosting of all tasks attached to this group. + +The current schedtune controller implementation is really simple and has these +main characteristics: + + 1) It is only possible to create 1 level depth hierarchies + + The root control groups define the system-wide boost value to be applied + by default to all tasks. Its direct subgroups are named "boost groups" and + they define the boost value for specific set of tasks. + Further nested subgroups are not allowed since they do not have a sensible + meaning from a user-space standpoint. + + 2) It is possible to define only a limited number of "boost groups" + + This number is defined at compile time and by default configured to 16. + This is a design decision motivated by two main reasons: + a) In a real system we do not expect utilization scenarios with more then few + boost groups. For example, a reasonable collection of groups could be + just "background", "interactive" and "performance". + b) It simplifies the implementation considerably, especially for the code + which has to compute the per CPU boosting once there are multiple + RUNNABLE tasks with different boost values. + +Such a simple design should allow servicing the main utilization scenarios identified +so far. It provides a simple interface which can be used to manage the +power-performance of all tasks or only selected tasks. +Moreover, this interface can be easily integrated by user-space run-times (e.g. +Android, ChromeOS) to implement a QoS solution for task boosting based on tasks +classification, which has been a long standing requirement. + +Setup and usage +--------------- + +0. Use a kernel with CGROUP_SCHEDTUNE support enabled + +1. Check that the "schedtune" CGroup controller is available: + + root@linaro-nano:~# cat /proc/cgroups + #subsys_name hierarchy num_cgroups enabled + cpuset 0 1 1 + cpu 0 1 1 + schedtune 0 1 1 + +2. Mount a tmpfs to create the CGroups mount point (Optional) + + root@linaro-nano:~# sudo mount -t tmpfs cgroups /sys/fs/cgroup + +3. Mount the "schedtune" controller + + root@linaro-nano:~# mkdir /sys/fs/cgroup/stune + root@linaro-nano:~# sudo mount -t cgroup -o schedtune stune /sys/fs/cgroup/stune + +4. Setup the system-wide boost value (Optional) + + If not configured the root control group has a 0% boost value, which + basically disables boosting for all tasks in the system thus running in + an energy-efficient mode. + + root@linaro-nano:~# echo $SYSBOOST > /sys/fs/cgroup/stune/schedtune.boost + +5. Create task groups and configure their specific boost value (Optional) + + For example here we create a "performance" boost group configure to boost + all its tasks to 100% + + root@linaro-nano:~# mkdir /sys/fs/cgroup/stune/performance + root@linaro-nano:~# echo 100 > /sys/fs/cgroup/stune/performance/schedtune.boost + +6. Move tasks into the boost group + + For example, the following moves the tasks with PID $TASKPID (and all its + threads) into the "performance" boost group. + + root@linaro-nano:~# echo "TASKPID > /sys/fs/cgroup/stune/performance/cgroup.procs + +This simple configuration allows only the threads of the $TASKPID task to run, +when needed, at the highest OPP in the most capable CPU of the system. + + +6. Question and Answers +======================= + +What about "auto" mode? +----------------------- + +The 'auto' mode as described in [5] can be implemented by interfacing SchedTune +with some suitable user-space element. This element could use the exposed +system-wide or cgroup based interface. + +How are multiple groups of tasks with different boost values managed? +--------------------------------------------------------------------- + +The current SchedTune implementation keeps track of the boosted RUNNABLE tasks +on a CPU. Once sched-DVFS selects the OPP to run a CPU at, the CPU utilization +is boosted with a value which is the maximum of the boost values of the +currently RUNNABLE tasks in its RQ. + +This allows sched-DVFS to boost a CPU only while there are boosted tasks ready +to run and switch back to the energy efficient mode as soon as the last boosted +task is dequeued. + + +7. References +============= +[1] http://lwn.net/Articles/552889 +[2] http://lkml.org/lkml/2012/5/18/91 +[3] http://lkml.org/lkml/2015/6/26/620 + From a9c3e5d96fb771ac41fd01b606e9fc46bd9454b0 Mon Sep 17 00:00:00 2001 From: Michael Turquette Date: Mon, 22 Feb 2016 17:22:42 -0800 Subject: [PATCH 47/86] FROMLIST: cpufreq: introduce cpufreq_driver_is_slow Some architectures and platforms perform CPU frequency transitions through a non-blocking method, while some might block or sleep. Even when frequency transitions do not block or sleep they may be very slow. This distinction is important when trying to change frequency from a non-interruptible context in a scheduler hot path. Describe this distinction with a cpufreq driver flag, CPUFREQ_DRIVER_FAST. The default is to not have this flag set, thus erring on the side of caution. cpufreq_driver_is_slow() is also introduced in this patch. Setting the above flag will allow this function to return false. [smuckle@linaro.org: change flag/API to include drivers that are too slow for scheduler hot paths, in addition to those that block/sleep] Cc: Rafael J. Wysocki Cc: Viresh Kumar Signed-off-by: Michael Turquette Signed-off-by: Steve Muckle (am from https://patchwork.kernel.org/patch/8385731/) Change-Id: I6f1d929ee175da71d2939c00bded8aacbca37ec5 Signed-off-by: Ricky Liang --- drivers/cpufreq/cpufreq.c | 6 ++++++ include/linux/cpufreq.h | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index a0913ab299603..51e469d900fb4 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -118,6 +118,12 @@ bool have_governor_per_policy(void) } EXPORT_SYMBOL_GPL(have_governor_per_policy); +bool cpufreq_driver_is_slow(void) +{ + return !(cpufreq_driver->flags & CPUFREQ_DRIVER_FAST); +} +EXPORT_SYMBOL_GPL(cpufreq_driver_is_slow); + struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) { if (have_governor_per_policy()) diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index b7850b9cd62e0..78799917d052e 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -141,6 +141,7 @@ u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy); int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu); int cpufreq_update_policy(unsigned int cpu); bool have_governor_per_policy(void); +bool cpufreq_driver_is_slow(void); struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy); #else static inline unsigned int cpufreq_get(unsigned int cpu) @@ -253,6 +254,23 @@ struct cpufreq_driver { */ #define CPUFREQ_ASYNC_NOTIFICATION (1 << 4) +/* + * Set by drivers which want cpufreq core to check if CPU is running at a + * frequency present in freq-table exposed by the driver. For these drivers if + * CPU is found running at an out of table freq, we will try to set it to a freq + * from the table. And if that fails, we will stop further boot process by + * issuing a BUG_ON(). + */ +#define CPUFREQ_NEED_INITIAL_FREQ_CHECK (1 << 5) + +/* + * Indicates that it is safe to call cpufreq_driver_target from + * non-interruptable context in scheduler hot paths. Drivers must + * opt-in to this flag, as the safe default is that they might sleep + * or be too slow for hot path use. + */ +#define CPUFREQ_DRIVER_FAST (1 << 6) + int cpufreq_register_driver(struct cpufreq_driver *driver_data); int cpufreq_unregister_driver(struct cpufreq_driver *driver_data); From d45eed560faeac400ef27a679979276257946198 Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Fri, 19 Feb 2016 15:27:28 -0800 Subject: [PATCH 48/86] EAS: sched/fair: tunable to honor sync wakeups The EAS wakeup path currently ignores the sync flag. Sync wakeups are used with binder so the hint makes a difference when tasks are making lots of binder calls. --- include/linux/sched/sysctl.h | 2 ++ kernel/sched/fair.c | 5 ++++- kernel/sysctl.c | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index b8648441b7970..b3e9aab6f5b46 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -89,6 +89,8 @@ extern unsigned int sysctl_sched_new_task_windows; #define sysctl_sched_enable_hmp_task_placement 0 #endif /* CONFIG_SCHED_HMP */ +extern unsigned int sysctl_sched_is_big_little; +extern unsigned int sysctl_sched_sync_hint_enable; enum sched_tunable_scaling { SCHED_TUNABLESCALING_NONE, diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 74f4e8ee64ce2..7eeff5218ef29 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -56,6 +56,9 @@ unsigned int __read_mostly sysctl_sched_wake_to_idle = 0; unsigned int sysctl_sched_latency = 6000000ULL; unsigned int normalized_sysctl_sched_latency = 6000000ULL; +unsigned int sysctl_sched_is_big_little = 0; +unsigned int sysctl_sched_sync_hint_enable = 1; + /* * The initial- and re-scaling of tunables is configurable * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) @@ -5487,7 +5490,7 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync) int target_cpu = -1; int i; - if (sync) { + if (sysctl_sched_sync_hint_enable && sync) { int cpu = smp_processor_id(); cpumask_t search_cpus; cpumask_and(&search_cpus, tsk_cpus_allowed(p), cpu_online_mask); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 93e79b936328e..4b248b705dd85 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -477,6 +477,20 @@ static struct ctl_table kern_table[] = { .extra1 = &min_sched_granularity_ns, .extra2 = &max_sched_granularity_ns, }, + { + .procname = "sched_is_big_little", + .data = &sysctl_sched_is_big_little, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "sched_sync_hint_enable", + .data = &sysctl_sched_sync_hint_enable, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, { .procname = "sched_wakeup_granularity_ns", .data = &sysctl_sched_wakeup_granularity, From 051fab285a6bb27dd4e9b01e7cdbe0fa62874ae0 Mon Sep 17 00:00:00 2001 From: Srinath Sridharan Date: Wed, 2 Mar 2016 16:33:50 -0800 Subject: [PATCH 49/86] EAS: sched/fair: tunable to force selecting at cpu granularity EAS assumes that clusters with smaller capacity cores are more energy-efficient. This may not be true on non-big-little devices, so EAS can make incorrect cluster selections when finding a CPU to wake. The "sched_is_big_little" hint can be used to cause a cpu-based selection instead of cluster-based selection. --- kernel/sched/fair.c | 141 +++++++++++++++++++++++++++++++------------- 1 file changed, 100 insertions(+), 41 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 7eeff5218ef29..5dc6e46f041bb 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5482,6 +5482,58 @@ static int select_idle_sibling(struct task_struct *p, int target) return target; } +static inline int find_best_target(struct task_struct *p) +{ + int i; + int target_cpu = -1; + int target_capacity; + int idle_cpu = -1; + int backup_cpu = -1; + + /* + * Favor 1) busy cpu with most capacity at current OPP + * 2) busy cpu with capacity at higher OPP + * 3) idle_cpu with capacity at current OPP + */ + for_each_cpu(i, tsk_cpus_allowed(p)) { + + int cur_capacity = capacity_curr_of(i); + + /* + * p's blocked utilization is still accounted for on prev_cpu + * so prev_cpu will receive a negative bias due to the double + * accounting. However, the blocked utilization may be zero. + */ + int new_util = get_cpu_usage(i) + boosted_task_utilization(p); + + if (new_util > capacity_orig_of(i)) + continue; + + if (new_util < cur_capacity) { + + if (cpu_rq(i)->nr_running) { + if (target_capacity < cur_capacity) { + /* busy CPU with most capacity at current OPP */ + target_cpu = i; + target_capacity = cur_capacity; + } + } else if (idle_cpu < 0) { + /* first idle CPU with capacity at current OPP */ + idle_cpu = i; + } + } else if (backup_cpu < 0 && cpu_rq(i)->nr_running) { + /* first busy CPU with capacity at higher OPP */ + backup_cpu = i; + } + } + + if (target_cpu < 0) { + target_cpu = backup_cpu >= 0 ? backup_cpu : idle_cpu; + } + + return target_cpu; +} + static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync) { struct sched_domain *sd; @@ -5505,55 +5557,62 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync) sg = sd->groups; sg_target = sg; - - /* - * Find group with sufficient capacity. We only get here if no cpu is - * overutilized. We may end up overutilizing a cpu by adding the task, - * but that should not be any worse than select_idle_sibling(). - * load_balance() should sort it out later as we get above the tipping - * point. - */ - do { - /* Assuming all cpus are the same in group */ - int max_cap_cpu = group_first_cpu(sg); - + if (sysctl_sched_is_big_little) { /* - * Assume smaller max capacity means more energy-efficient. - * Ideally we should query the energy model for the right - * answer but it easily ends up in an exhaustive search. + * Find group with sufficient capacity. We only get here if no cpu is + * overutilized. We may end up overutilizing a cpu by adding the task, + * but that should not be any worse than select_idle_sibling(). + * load_balance() should sort it out later as we get above the tipping + * point. */ - if (capacity_of(max_cap_cpu) < target_max_cap && - task_fits_capacity(p, max_cap_cpu)) { - sg_target = sg; - target_max_cap = capacity_of(max_cap_cpu); - } + do { + /* Assuming all cpus are the same in group */ + int max_cap_cpu = group_first_cpu(sg); - } while (sg = sg->next, sg != sd->groups); + /* + * Assume smaller max capacity means more energy-efficient. + * Ideally we should query the energy model for the right + * answer but it easily ends up in an exhaustive search. + */ + if (capacity_of(max_cap_cpu) < target_max_cap && + task_fits_capacity(p, max_cap_cpu)) { + sg_target = sg; + target_max_cap = capacity_of(max_cap_cpu); + } - /* Find cpu with sufficient capacity */ - for_each_cpu_and(i, tsk_cpus_allowed(p), sched_group_cpus(sg_target)) { - /* - * p's blocked utilization is still accounted for on prev_cpu - * so prev_cpu will receive a negative bias due the double - * accouting. However, the blocked utilization may be zero. - */ - int new_usage = get_cpu_usage(i) + boosted_task_utilization(p); + } while (sg = sg->next, sg != sd->groups); - if (new_usage > capacity_orig_of(i)) - continue; + /* Find cpu with sufficient capacity */ + for_each_cpu_and(i, tsk_cpus_allowed(p), sched_group_cpus(sg_target)) { + /* + * p's blocked utilization is still accounted for on prev_cpu + * so prev_cpu will receive a negative bias due the double + * accouting. However, the blocked utilization may be zero. + */ + int new_usage = get_cpu_usage(i) + boosted_task_utilization(p); - if (new_usage < capacity_curr_of(i)) { - target_cpu = i; - if (cpu_rq(i)->nr_running) - break; - } + if (new_usage > capacity_orig_of(i)) + continue; - /* cpu has capacity at higher OPP, keep it as fallback */ - if (target_cpu < 0) - target_cpu = i; - } + if (new_usage < capacity_curr_of(i)) { + target_cpu = i; + if (cpu_rq(i)->nr_running) + break; + } + + /* cpu has capacity at higher OPP, keep it as fallback */ + if (target_cpu < 0) + target_cpu = i; + } + } else { + /* + * Find a cpu with sufficient capacity + */ + int tmp_target = find_best_target(p); + if (tmp_target >= 0) target_cpu = tmp_target; + } - if (target_cpu < 0) + if (target_cpu < 0) target_cpu = task_cpu(p); else if (target_cpu != task_cpu(p)) { struct energy_env eenv = { From 9e9a86bc018dd14a9f3036e21efdb45603815e46 Mon Sep 17 00:00:00 2001 From: Riley Andrews Date: Wed, 21 Oct 2015 16:30:25 -0700 Subject: [PATCH 50/86] android: binder: Use wake up hint for synchronous transactions. (cherry pick from 572b57fc6f7fb6ffaa979d505ec2b0a9e9840cca) Use wake_up_interruptible_sync() to hint to the scheduler binder transactions are synchronous wakeups. Disable preemption while waking to avoid ping-ponging on the binder lock. Signed-off-by: Riley Andrews Bug: 30141999 Change-Id: If570d94ef3fed09c328052922d5a9e83d7ba479a --- drivers/android/binder.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index cd7dc1de32591..818afae541602 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -1706,8 +1706,15 @@ static void binder_transaction(struct binder_proc *proc, list_add_tail(&t->work.entry, target_list); tcomplete->type = BINDER_WORK_TRANSACTION_COMPLETE; list_add_tail(&tcomplete->entry, &thread->todo); - if (target_wait) - wake_up_interruptible(target_wait); + if (target_wait) { + if (reply || !(t->flags & TF_ONE_WAY)) { + preempt_disable(); + wake_up_interruptible_sync(target_wait); + sched_preempt_enable_no_resched(); + } else { + wake_up_interruptible(target_wait); + } + } return; err_get_unused_fd_failed: From 684a64646ba17551eed107ce619bc2e6c7c98c73 Mon Sep 17 00:00:00 2001 From: Riley Andrews Date: Wed, 21 Oct 2015 16:30:51 -0700 Subject: [PATCH 51/86] android: binder: Disable preemption while holding the global binder lock. (cherry pick from commit f681f0264fd2c51aa12190ff9be04622a7c8ca3f) Signed-off-by: Riley Andrews Bug: 30141999 Change-Id: I66ed44990d5c347d197e61dc49a37b5228c748d0 --- drivers/android/binder.c | 153 +++++++++++++++++++++++++++++---------- 1 file changed, 113 insertions(+), 40 deletions(-) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 818afae541602..e6c42e261c183 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -429,6 +429,7 @@ static inline void binder_lock(const char *tag) { trace_binder_lock(tag); mutex_lock(&binder_main_lock); + preempt_disable(); trace_binder_locked(tag); } @@ -436,8 +437,62 @@ static inline void binder_unlock(const char *tag) { trace_binder_unlock(tag); mutex_unlock(&binder_main_lock); + preempt_enable(); } +static inline void *kzalloc_preempt_disabled(size_t size) +{ + void *ptr; + + ptr = kzalloc(size, GFP_NOWAIT); + if (ptr) + return ptr; + + preempt_enable_no_resched(); + ptr = kzalloc(size, GFP_KERNEL); + preempt_disable(); + + return ptr; +} + +static inline long copy_to_user_preempt_disabled(void __user *to, const void *from, long n) +{ + long ret; + + preempt_enable_no_resched(); + ret = copy_to_user(to, from, n); + preempt_disable(); + return ret; +} + +static inline long copy_from_user_preempt_disabled(void *to, const void __user *from, long n) +{ + long ret; + + preempt_enable_no_resched(); + ret = copy_from_user(to, from, n); + preempt_disable(); + return ret; +} + +#define get_user_preempt_disabled(x, ptr) \ +({ \ + int __ret; \ + preempt_enable_no_resched(); \ + __ret = get_user(x, ptr); \ + preempt_disable(); \ + __ret; \ +}) + +#define put_user_preempt_disabled(x, ptr) \ +({ \ + int __ret; \ + preempt_enable_no_resched(); \ + __ret = put_user(x, ptr); \ + preempt_disable(); \ + __ret; \ +}) + static void binder_set_nice(long nice) { long min_nice; @@ -571,6 +626,8 @@ static int binder_update_page_range(struct binder_proc *proc, int allocate, else mm = get_task_mm(proc->tsk); + preempt_enable_no_resched(); + if (mm) { down_write(&mm->mmap_sem); vma = proc->vma; @@ -626,6 +683,9 @@ static int binder_update_page_range(struct binder_proc *proc, int allocate, up_write(&mm->mmap_sem); mmput(mm); } + + preempt_disable(); + return 0; free_range: @@ -648,6 +708,9 @@ static int binder_update_page_range(struct binder_proc *proc, int allocate, up_write(&mm->mmap_sem); mmput(mm); } + + preempt_disable(); + return -ENOMEM; } @@ -907,7 +970,7 @@ static struct binder_node *binder_new_node(struct binder_proc *proc, return NULL; } - node = kzalloc(sizeof(*node), GFP_KERNEL); + node = kzalloc_preempt_disabled(sizeof(*node)); if (node == NULL) return NULL; binder_stats_created(BINDER_STAT_NODE); @@ -1048,7 +1111,7 @@ static struct binder_ref *binder_get_ref_for_node(struct binder_proc *proc, else return ref; } - new_ref = kzalloc(sizeof(*ref), GFP_KERNEL); + new_ref = kzalloc_preempt_disabled(sizeof(*ref)); if (new_ref == NULL) return NULL; binder_stats_created(BINDER_STAT_REF); @@ -1445,14 +1508,14 @@ static void binder_transaction(struct binder_proc *proc, e->to_proc = target_proc->pid; /* TODO: reuse incoming transaction for reply */ - t = kzalloc(sizeof(*t), GFP_KERNEL); + t = kzalloc_preempt_disabled(sizeof(*t)); if (t == NULL) { return_error = BR_FAILED_REPLY; goto err_alloc_t_failed; } binder_stats_created(BINDER_STAT_TRANSACTION); - tcomplete = kzalloc(sizeof(*tcomplete), GFP_KERNEL); + tcomplete = kzalloc_preempt_disabled(sizeof(*tcomplete)); if (tcomplete == NULL) { return_error = BR_FAILED_REPLY; goto err_alloc_tcomplete_failed; @@ -1509,14 +1572,14 @@ static void binder_transaction(struct binder_proc *proc, offp = (binder_size_t *)(t->buffer->data + ALIGN(tr->data_size, sizeof(void *))); - if (copy_from_user(t->buffer->data, (const void __user *)(uintptr_t) + if (copy_from_user_preempt_disabled(t->buffer->data, (const void __user *)(uintptr_t) tr->data.ptr.buffer, tr->data_size)) { binder_user_error("%d:%d got transaction with invalid data ptr\n", proc->pid, thread->pid); return_error = BR_FAILED_REPLY; goto err_copy_data_failed; } - if (copy_from_user(offp, (const void __user *)(uintptr_t) + if (copy_from_user_preempt_disabled(offp, (const void __user *)(uintptr_t) tr->data.ptr.offsets, tr->offsets_size)) { binder_user_error("%d:%d got transaction with invalid offsets ptr\n", proc->pid, thread->pid); @@ -1708,9 +1771,7 @@ static void binder_transaction(struct binder_proc *proc, list_add_tail(&tcomplete->entry, &thread->todo); if (target_wait) { if (reply || !(t->flags & TF_ONE_WAY)) { - preempt_disable(); wake_up_interruptible_sync(target_wait); - sched_preempt_enable_no_resched(); } else { wake_up_interruptible(target_wait); } @@ -1773,7 +1834,7 @@ int binder_thread_write(struct binder_proc *proc, void __user *end = buffer + size; while (ptr < end && thread->return_error == BR_OK) { - if (get_user(cmd, (uint32_t __user *)ptr)) + if (get_user_preempt_disabled(cmd, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); trace_binder_command(cmd); @@ -1791,7 +1852,7 @@ int binder_thread_write(struct binder_proc *proc, struct binder_ref *ref; const char *debug_string; - if (get_user(target, (uint32_t __user *)ptr)) + if (get_user_preempt_disabled(target, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); if (target == 0 && binder_context_mgr_node && @@ -1850,10 +1911,10 @@ int binder_thread_write(struct binder_proc *proc, binder_uintptr_t cookie; struct binder_node *node; - if (get_user(node_ptr, (binder_uintptr_t __user *)ptr)) + if (get_user_preempt_disabled(node_ptr, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); - if (get_user(cookie, (binder_uintptr_t __user *)ptr)) + if (get_user_preempt_disabled(cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); node = binder_get_node(proc, node_ptr); @@ -1911,7 +1972,7 @@ int binder_thread_write(struct binder_proc *proc, binder_uintptr_t data_ptr; struct binder_buffer *buffer; - if (get_user(data_ptr, (binder_uintptr_t __user *)ptr)) + if (get_user_preempt_disabled(data_ptr, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); @@ -1953,7 +2014,7 @@ int binder_thread_write(struct binder_proc *proc, case BC_REPLY: { struct binder_transaction_data tr; - if (copy_from_user(&tr, ptr, sizeof(tr))) + if (copy_from_user_preempt_disabled(&tr, ptr, sizeof(tr))) return -EFAULT; ptr += sizeof(tr); binder_transaction(proc, thread, &tr, cmd == BC_REPLY); @@ -2003,10 +2064,10 @@ int binder_thread_write(struct binder_proc *proc, struct binder_ref *ref; struct binder_ref_death *death; - if (get_user(target, (uint32_t __user *)ptr)) + if (get_user_preempt_disabled(target, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); - if (get_user(cookie, (binder_uintptr_t __user *)ptr)) + if (get_user_preempt_disabled(cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); ref = binder_get_ref(proc, target, false); @@ -2035,7 +2096,7 @@ int binder_thread_write(struct binder_proc *proc, proc->pid, thread->pid); break; } - death = kzalloc(sizeof(*death), GFP_KERNEL); + death = kzalloc_preempt_disabled(sizeof(*death)); if (death == NULL) { thread->return_error = BR_ERROR; binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, @@ -2090,7 +2151,7 @@ int binder_thread_write(struct binder_proc *proc, binder_uintptr_t cookie; struct binder_ref_death *death = NULL; - if (get_user(cookie, (binder_uintptr_t __user *)ptr)) + if (get_user_preempt_disabled(cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(void *); @@ -2170,7 +2231,7 @@ static int binder_thread_read(struct binder_proc *proc, int wait_for_proc_work; if (*consumed == 0) { - if (put_user(BR_NOOP, (uint32_t __user *)ptr)) + if (put_user_preempt_disabled(BR_NOOP, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); } @@ -2181,7 +2242,7 @@ static int binder_thread_read(struct binder_proc *proc, if (thread->return_error != BR_OK && ptr < end) { if (thread->return_error2 != BR_OK) { - if (put_user(thread->return_error2, (uint32_t __user *)ptr)) + if (put_user_preempt_disabled(thread->return_error2, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); binder_stat_br(proc, thread, thread->return_error2); @@ -2189,7 +2250,7 @@ static int binder_thread_read(struct binder_proc *proc, goto done; thread->return_error2 = BR_OK; } - if (put_user(thread->return_error, (uint32_t __user *)ptr)) + if (put_user_preempt_disabled(thread->return_error, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); binder_stat_br(proc, thread, thread->return_error); @@ -2273,7 +2334,7 @@ static int binder_thread_read(struct binder_proc *proc, } break; case BINDER_WORK_TRANSACTION_COMPLETE: { cmd = BR_TRANSACTION_COMPLETE; - if (put_user(cmd, (uint32_t __user *)ptr)) + if (put_user_preempt_disabled(cmd, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); @@ -2315,14 +2376,14 @@ static int binder_thread_read(struct binder_proc *proc, node->has_weak_ref = 0; } if (cmd != BR_NOOP) { - if (put_user(cmd, (uint32_t __user *)ptr)) + if (put_user_preempt_disabled(cmd, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); - if (put_user(node->ptr, + if (put_user_preempt_disabled(node->ptr, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); - if (put_user(node->cookie, + if (put_user_preempt_disabled(node->cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); @@ -2366,10 +2427,10 @@ static int binder_thread_read(struct binder_proc *proc, cmd = BR_CLEAR_DEATH_NOTIFICATION_DONE; else cmd = BR_DEAD_BINDER; - if (put_user(cmd, (uint32_t __user *)ptr)) + if (put_user_preempt_disabled(cmd, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); - if (put_user(death->cookie, + if (put_user_preempt_disabled(death->cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); @@ -2437,10 +2498,10 @@ static int binder_thread_read(struct binder_proc *proc, ALIGN(t->buffer->data_size, sizeof(void *)); - if (put_user(cmd, (uint32_t __user *)ptr)) + if (put_user_preempt_disabled(cmd, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); - if (copy_to_user(ptr, &tr, sizeof(tr))) + if (copy_to_user_preempt_disabled(ptr, &tr, sizeof(tr))) return -EFAULT; ptr += sizeof(tr); @@ -2482,7 +2543,7 @@ static int binder_thread_read(struct binder_proc *proc, binder_debug(BINDER_DEBUG_THREADS, "%d:%d BR_SPAWN_LOOPER\n", proc->pid, thread->pid); - if (put_user(BR_SPAWN_LOOPER, (uint32_t __user *)buffer)) + if (put_user_preempt_disabled(BR_SPAWN_LOOPER, (uint32_t __user *)buffer)) return -EFAULT; binder_stat_br(proc, thread, BR_SPAWN_LOOPER); } @@ -2557,7 +2618,7 @@ static struct binder_thread *binder_get_thread(struct binder_proc *proc) break; } if (*p == NULL) { - thread = kzalloc(sizeof(*thread), GFP_KERNEL); + thread = kzalloc_preempt_disabled(sizeof(*thread)); if (thread == NULL) return NULL; binder_stats_created(BINDER_STAT_THREAD); @@ -2661,7 +2722,7 @@ static int binder_ioctl_write_read(struct file *filp, ret = -EINVAL; goto out; } - if (copy_from_user(&bwr, ubuf, sizeof(bwr))) { + if (copy_from_user_preempt_disabled(&bwr, ubuf, sizeof(bwr))) { ret = -EFAULT; goto out; } @@ -2679,7 +2740,7 @@ static int binder_ioctl_write_read(struct file *filp, trace_binder_write_done(ret); if (ret < 0) { bwr.read_consumed = 0; - if (copy_to_user(ubuf, &bwr, sizeof(bwr))) + if (copy_to_user_preempt_disabled(ubuf, &bwr, sizeof(bwr))) ret = -EFAULT; goto out; } @@ -2693,7 +2754,7 @@ static int binder_ioctl_write_read(struct file *filp, if (!list_empty(&proc->todo)) wake_up_interruptible(&proc->wait); if (ret < 0) { - if (copy_to_user(ubuf, &bwr, sizeof(bwr))) + if (copy_to_user_preempt_disabled(ubuf, &bwr, sizeof(bwr))) ret = -EFAULT; goto out; } @@ -2703,7 +2764,7 @@ static int binder_ioctl_write_read(struct file *filp, proc->pid, thread->pid, (u64)bwr.write_consumed, (u64)bwr.write_size, (u64)bwr.read_consumed, (u64)bwr.read_size); - if (copy_to_user(ubuf, &bwr, sizeof(bwr))) { + if (copy_to_user_preempt_disabled(ubuf, &bwr, sizeof(bwr))) { ret = -EFAULT; goto out; } @@ -2781,7 +2842,7 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) goto err; break; case BINDER_SET_MAX_THREADS: - if (copy_from_user(&proc->max_threads, ubuf, sizeof(proc->max_threads))) { + if (copy_from_user_preempt_disabled(&proc->max_threads, ubuf, sizeof(proc->max_threads))) { ret = -EINVAL; goto err; } @@ -2804,7 +2865,8 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) ret = -EINVAL; goto err; } - if (put_user(BINDER_CURRENT_PROTOCOL_VERSION, + + if (put_user_preempt_disabled(BINDER_CURRENT_PROTOCOL_VERSION, &ver->protocol_version)) { ret = -EINVAL; goto err; @@ -2870,6 +2932,7 @@ static struct vm_operations_struct binder_vm_ops = { static int binder_mmap(struct file *filp, struct vm_area_struct *vma) { int ret; + struct vm_struct *area; struct binder_proc *proc = filp->private_data; const char *failure_string; @@ -2930,7 +2993,11 @@ static int binder_mmap(struct file *filp, struct vm_area_struct *vma) vma->vm_ops = &binder_vm_ops; vma->vm_private_data = proc; - if (binder_update_page_range(proc, 1, proc->buffer, proc->buffer + PAGE_SIZE, vma)) { + /* binder_update_page_range assumes preemption is disabled */ + preempt_disable(); + ret = binder_update_page_range(proc, 1, proc->buffer, proc->buffer + PAGE_SIZE, vma); + preempt_enable_no_resched(); + if (ret) { ret = -ENOMEM; failure_string = "alloc small buf"; goto err_alloc_small_buf_failed; @@ -3200,8 +3267,12 @@ static void binder_deferred_func(struct work_struct *work) int defer; do { - binder_lock(__func__); + trace_binder_lock(__func__); + mutex_lock(&binder_main_lock); + trace_binder_locked(__func__); + mutex_lock(&binder_deferred_lock); + preempt_disable(); if (!hlist_empty(&binder_deferred_list)) { proc = hlist_entry(binder_deferred_list.first, struct binder_proc, deferred_work_node); @@ -3227,7 +3298,9 @@ static void binder_deferred_func(struct work_struct *work) if (defer & BINDER_DEFERRED_RELEASE) binder_deferred_release(proc); /* frees proc */ - binder_unlock(__func__); + trace_binder_unlock(__func__); + mutex_unlock(&binder_main_lock); + preempt_enable_no_resched(); if (files) put_files_struct(files); } while (proc); From 8ce7ce259e216429a82d94fb1d5eb3529f295b0f Mon Sep 17 00:00:00 2001 From: Stephen Smalley Date: Mon, 5 Nov 2012 08:15:34 -0500 Subject: [PATCH 52/86] Add security hooks to binder and implement the hooks for SELinux. Add security hooks to the binder and implement the hooks for SELinux. The security hooks enable security modules such as SELinux to implement controls over binder IPC. The security hooks include support for controlling what process can become the binder context manager (binder_set_context_mgr), controlling the ability of a process to invoke a binder transaction/IPC to another process (binder_transaction), controlling the ability a process to transfer a binder reference to another process (binder_transfer_binder), and controlling the ability of a process to transfer an open file to another process (binder_transfer_file). This support is used by SE Android, http://selinuxproject.org/page/SEAndroid. Change-Id: I34266b66320b6a3df9ac01833d7f94daf742920e Signed-off-by: Stephen Smalley --- drivers/android/binder.c | 21 +++++++++ include/linux/security.h | 29 +++++++++++++ security/capability.c | 24 +++++++++++ security/security.c | 20 +++++++++ security/selinux/hooks.c | 66 +++++++++++++++++++++++++++++ security/selinux/include/classmap.h | 1 + 6 files changed, 161 insertions(+) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index e6c42e261c183..bf411a82a12a9 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -37,6 +37,7 @@ #include #include #include +#include #ifdef CONFIG_ANDROID_BINDER_IPC_32BIT #define BINDER_IPC_32BIT 1 @@ -1477,6 +1478,10 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_DEAD_REPLY; goto err_dead_binder; } + if (security_binder_transaction(proc->tsk, target_proc->tsk) < 0) { + return_error = BR_FAILED_REPLY; + goto err_invalid_target_handle; + } if (!(tr->flags & TF_ONE_WAY) && thread->transaction_stack) { struct binder_transaction *tmp; @@ -1634,6 +1639,10 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_binder_get_ref_for_node_failed; } + if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) { + return_error = BR_FAILED_REPLY; + goto err_binder_get_ref_for_node_failed; + } ref = binder_get_ref_for_node(target_proc, node); if (ref == NULL) { return_error = BR_FAILED_REPLY; @@ -1667,6 +1676,10 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_binder_get_ref_failed; } + if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) { + return_error = BR_FAILED_REPLY; + goto err_binder_get_ref_failed; + } if (ref->node->proc == target_proc) { if (fp->type == BINDER_TYPE_HANDLE) fp->type = BINDER_TYPE_BINDER; @@ -1726,6 +1739,11 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_fget_failed; } + if (security_binder_transfer_file(proc->tsk, target_proc->tsk, file) < 0) { + fput(file); + return_error = BR_FAILED_REPLY; + goto err_get_unused_fd_failed; + } target_fd = task_get_unused_fd_flags(target_proc, O_CLOEXEC); if (target_fd < 0) { fput(file); @@ -2783,6 +2801,9 @@ static int binder_ioctl_set_ctx_mgr(struct file *filp) ret = -EBUSY; goto out; } + ret = security_binder_set_context_mgr(proc->tsk); + if (ret < 0) + goto out; if (uid_valid(binder_context_mgr_uid)) { if (!uid_eq(binder_context_mgr_uid, curr_euid)) { pr_err("BINDER_SET_CONTEXT_MGR bad uid %d != %d\n", diff --git a/include/linux/security.h b/include/linux/security.h index c1f7c807ae868..4b5df69059d53 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -1403,6 +1403,11 @@ static inline void security_free_mnt_opts(struct security_mnt_opts *opts) struct security_operations { char name[SECURITY_NAME_MAX + 1]; + int (*binder_set_context_mgr) (struct task_struct *mgr); + int (*binder_transaction) (struct task_struct *from, struct task_struct *to); + int (*binder_transfer_binder) (struct task_struct *from, struct task_struct *to); + int (*binder_transfer_file) (struct task_struct *from, struct task_struct *to, struct file *file); + int (*ptrace_access_check) (struct task_struct *child, unsigned int mode); int (*ptrace_traceme) (struct task_struct *parent); int (*capget) (struct task_struct *target, @@ -1695,6 +1700,10 @@ extern void __init security_fixup_ops(struct security_operations *ops); /* Security operations */ +int security_binder_set_context_mgr(struct task_struct *mgr); +int security_binder_transaction(struct task_struct *from, struct task_struct *to); +int security_binder_transfer_binder(struct task_struct *from, struct task_struct *to); +int security_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file); int security_ptrace_access_check(struct task_struct *child, unsigned int mode); int security_ptrace_traceme(struct task_struct *parent); int security_capget(struct task_struct *target, @@ -1879,6 +1888,26 @@ static inline int security_init(void) return 0; } +static inline int security_binder_set_context_mgr(struct task_struct *mgr) +{ + return 0; +} + +static inline int security_binder_transaction(struct task_struct *from, struct task_struct *to) +{ + return 0; +} + +static inline int security_binder_transfer_binder(struct task_struct *from, struct task_struct *to) +{ + return 0; +} + +static inline int security_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) +{ + return 0; +} + static inline int security_ptrace_access_check(struct task_struct *child, unsigned int mode) { diff --git a/security/capability.c b/security/capability.c index 1728d4e375db5..6e4fc776badf1 100644 --- a/security/capability.c +++ b/security/capability.c @@ -12,6 +12,26 @@ #include +static int cap_binder_set_context_mgr(struct task_struct *mgr) +{ + return 0; +} + +static int cap_binder_transaction(struct task_struct *from, struct task_struct *to) +{ + return 0; +} + +static int cap_binder_transfer_binder(struct task_struct *from, struct task_struct *to) +{ + return 0; +} + +static int cap_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) +{ + return 0; +} + static int cap_syslog(int type) { return 0; @@ -903,6 +923,10 @@ static void cap_audit_rule_free(void *lsmrule) void __init security_fixup_ops(struct security_operations *ops) { + set_to_cap_if_null(ops, binder_set_context_mgr); + set_to_cap_if_null(ops, binder_transaction); + set_to_cap_if_null(ops, binder_transfer_binder); + set_to_cap_if_null(ops, binder_transfer_file); set_to_cap_if_null(ops, ptrace_access_check); set_to_cap_if_null(ops, ptrace_traceme); set_to_cap_if_null(ops, capget); diff --git a/security/security.c b/security/security.c index 12380e7eb76f4..ea34b97abe44b 100644 --- a/security/security.c +++ b/security/security.c @@ -134,6 +134,26 @@ int __init register_security(struct security_operations *ops) /* Security operations */ +int security_binder_set_context_mgr(struct task_struct *mgr) +{ + return security_ops->binder_set_context_mgr(mgr); +} + +int security_binder_transaction(struct task_struct *from, struct task_struct *to) +{ + return security_ops->binder_transaction(from, to); +} + +int security_binder_transfer_binder(struct task_struct *from, struct task_struct *to) +{ + return security_ops->binder_transfer_binder(from, to); +} + +int security_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) +{ + return security_ops->binder_transfer_file(from, to, file); +} + int security_ptrace_access_check(struct task_struct *child, unsigned int mode) { #ifdef CONFIG_SECURITY_YAMA_STACKED diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 29d96007f9d56..626533313cf57 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -1871,6 +1871,67 @@ static inline u32 open_file_to_av(struct file *file) /* Hook functions begin here. */ +static int selinux_binder_set_context_mgr(struct task_struct *mgr) +{ + u32 mysid = current_sid(); + u32 mgrsid = task_sid(mgr); + + return avc_has_perm(mysid, mgrsid, SECCLASS_BINDER, BINDER__SET_CONTEXT_MGR, NULL); +} + +static int selinux_binder_transaction(struct task_struct *from, struct task_struct *to) +{ + u32 mysid = current_sid(); + u32 fromsid = task_sid(from); + u32 tosid = task_sid(to); + int rc; + + if (mysid != fromsid) { + rc = avc_has_perm(mysid, fromsid, SECCLASS_BINDER, BINDER__IMPERSONATE, NULL); + if (rc) + return rc; + } + + return avc_has_perm(fromsid, tosid, SECCLASS_BINDER, BINDER__CALL, NULL); +} + +static int selinux_binder_transfer_binder(struct task_struct *from, struct task_struct *to) +{ + u32 fromsid = task_sid(from); + u32 tosid = task_sid(to); + return avc_has_perm(fromsid, tosid, SECCLASS_BINDER, BINDER__TRANSFER, NULL); +} + +static int selinux_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) +{ + u32 sid = task_sid(to); + struct file_security_struct *fsec = file->f_security; + struct inode *inode = file->f_path.dentry->d_inode; + struct inode_security_struct *isec = inode->i_security; + struct common_audit_data ad; + struct selinux_audit_data sad = {0,}; + int rc; + + ad.type = LSM_AUDIT_DATA_PATH; + ad.u.path = file->f_path; + ad.selinux_audit_data = &sad; + + if (sid != fsec->sid) { + rc = avc_has_perm(sid, fsec->sid, + SECCLASS_FD, + FD__USE, + &ad); + if (rc) + return rc; + } + + if (unlikely(IS_PRIVATE(inode))) + return 0; + + return avc_has_perm(sid, isec->sid, isec->sclass, file_to_av(file), + &ad); +} + static int selinux_ptrace_access_check(struct task_struct *child, unsigned int mode) { @@ -5717,6 +5778,11 @@ static int selinux_key_getsecurity(struct key *key, char **_buffer) static struct security_operations selinux_ops = { .name = "selinux", + .binder_set_context_mgr = selinux_binder_set_context_mgr, + .binder_transaction = selinux_binder_transaction, + .binder_transfer_binder = selinux_binder_transfer_binder, + .binder_transfer_file = selinux_binder_transfer_file, + .ptrace_access_check = selinux_ptrace_access_check, .ptrace_traceme = selinux_ptrace_traceme, .capget = selinux_capget, diff --git a/security/selinux/include/classmap.h b/security/selinux/include/classmap.h index 14d04e63b1f0e..c32ff7bde81ab 100644 --- a/security/selinux/include/classmap.h +++ b/security/selinux/include/classmap.h @@ -151,5 +151,6 @@ struct security_class_mapping secclass_map[] = { { "kernel_service", { "use_as_override", "create_files_as", NULL } }, { "tun_socket", { COMMON_SOCK_PERMS, "attach_queue", NULL } }, + { "binder", { "impersonate", "call", "set_context_mgr", "transfer", NULL } }, { NULL } }; From d9da179c6fc10829af407dfef3855e7db368a2b0 Mon Sep 17 00:00:00 2001 From: Andrew Bresticker Date: Fri, 23 Oct 2015 15:13:42 -0700 Subject: [PATCH 53/86] CHROMIUM: android: binder: Fix potential scheduling-while-atomic (cherry picked from commit 166b45af97359159f9585a836c9849e725e31fd6) Commit f1e7f0a724f6 ("android: binder: Disable preemption while holding the global binder lock.") re-enabled preemption around most of the sites where calls to potentially sleeping functions were made, but missed __alloc_fd(), which can sleep if the fdtable needs to be resized. Re-enable preemption around __alloc_fd() as well as __fd_install() which can now sleep in upstream kernels as of commit 8a81252b774b ("fs/file.c: don't acquire files->file_lock in fd_install()"). BUG=chrome-os-partner:44012 TEST=Build and boot on Smaug. Change-Id: I9819c4b95876f697e75b1b84810b6c520d9c33ec Signed-off-by: Andrew Bresticker Reviewed-on: https://chromium-review.googlesource.com/308582 Reviewed-by: Stephen Barber Reviewed-by: Riley Andrews Bug: 30141999 --- drivers/android/binder.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index bf411a82a12a9..f0a8a67f6a1cf 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -382,6 +382,7 @@ static int task_get_unused_fd_flags(struct binder_proc *proc, int flags) struct files_struct *files = proc->files; unsigned long rlim_cur; unsigned long irqs; + int ret; if (files == NULL) return -ESRCH; @@ -392,7 +393,11 @@ static int task_get_unused_fd_flags(struct binder_proc *proc, int flags) rlim_cur = task_rlimit(proc->tsk, RLIMIT_NOFILE); unlock_task_sighand(proc->tsk, &irqs); - return __alloc_fd(files, 0, rlim_cur, flags); + preempt_enable_no_resched(); + ret = __alloc_fd(files, 0, rlim_cur, flags); + preempt_disable(); + + return ret; } /* @@ -401,8 +406,11 @@ static int task_get_unused_fd_flags(struct binder_proc *proc, int flags) static void task_fd_install( struct binder_proc *proc, unsigned int fd, struct file *file) { - if (proc->files) + if (proc->files) { + preempt_enable_no_resched(); __fd_install(proc->files, fd, file); + preempt_disable(); + } } /* From 3c0b32c8fb98c820889b2abdaaa8ea2d1fb4fe14 Mon Sep 17 00:00:00 2001 From: Lisa Du Date: Wed, 17 Feb 2016 09:32:52 +0800 Subject: [PATCH 54/86] UPSTREAM: drivers: android: correct the size of struct binder_uintptr_t for BC_DEAD_BINDER_DONE There's one point was missed in the patch commit da49889deb34 ("staging: binder: Support concurrent 32 bit and 64 bit processes."). When configure BINDER_IPC_32BIT, the size of binder_uintptr_t was 32bits, but size of void * is 64bit on 64bit system. Correct it here. Signed-off-by: Lisa Du Signed-off-by: Nicolas Boichat Fixes: da49889deb34 ("staging: binder: Support concurrent 32 bit and 64 bit processes.") Cc: Acked-by: Olof Johansson Signed-off-by: Greg Kroah-Hartman (cherry picked from commit 7a64cd887fdb97f074c3fda03bee0bfb9faceac3) BUG=b:26833439 TEST=See b:26833439 comment #22 Signed-off-by: Nicolas Boichat Change-Id: I204b074fd8cad74cfbeaf322fcdc976877736396 --- drivers/android/binder.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index f0a8a67f6a1cf..b409f1fb19072 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -2180,7 +2180,7 @@ int binder_thread_write(struct binder_proc *proc, if (get_user_preempt_disabled(cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; - ptr += sizeof(void *); + ptr += sizeof(cookie); list_for_each_entry(w, &proc->delivered_death, entry) { struct binder_ref_death *tmp_death = container_of(w, struct binder_ref_death, work); From 5b33a29eb3e53c820a9904189abf84d1cf7cd625 Mon Sep 17 00:00:00 2001 From: Alex Naidis Date: Wed, 5 Jul 2017 16:50:59 +0200 Subject: [PATCH 55/86] binder: Disable debug mask According to Google we should set this to 0 as there is excessive logging in specific usecases which has a negative impact on latency. Change-Id: If03401694c2b746b5e2111cb5daafa8eab169e29 Signed-off-by: Alex Naidis Reviewed-on: https://gerrit.nextbit.com:8443/21174 Reviewed-by: William McVicker Signed-off-by: Joe Maples --- drivers/android/binder.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index b409f1fb19072..df98ea84c2502 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -112,8 +112,7 @@ enum { BINDER_DEBUG_PRIORITY_CAP = 1U << 14, BINDER_DEBUG_BUFFER_ALLOC_ASYNC = 1U << 15, }; -static uint32_t binder_debug_mask; - +static uint32_t binder_debug_mask = 0; module_param_named(debug_mask, binder_debug_mask, uint, S_IWUSR | S_IRUGO); static bool binder_debug_no_lock; From ef67dde89855007b72c270ec8200e01dc789e273 Mon Sep 17 00:00:00 2001 From: Riley Andrews Date: Tue, 1 Sep 2015 12:42:07 -0700 Subject: [PATCH 56/86] android: binder: Use wake up hint for synchronous transactions. Use wake_up_interruptible_sync() to hint to the scheduler binder transactions are synchronous wakeups. Disable premption while waking to avoid ping-ponging on the binder lock. Change-Id: Ic406a232d0873662f80148e37acefe5243d912a0 Signed-off-by: Joe Maples --- drivers/android/binder.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index df98ea84c2502..e2ef66adf5197 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -1796,8 +1796,11 @@ static void binder_transaction(struct binder_proc *proc, list_add_tail(&tcomplete->entry, &thread->todo); if (target_wait) { if (reply || !(t->flags & TF_ONE_WAY)) { + preempt_disable(); wake_up_interruptible_sync(target_wait); - } else { + sched_preempt_enable_no_resched(); + } + else { wake_up_interruptible(target_wait); } } From bc3c0e98153ff4b175b120024a229587ecb314e4 Mon Sep 17 00:00:00 2001 From: "www.anuprita804@gmail.com" Date: Fri, 18 Aug 2017 17:56:28 +0200 Subject: [PATCH 57/86] test --- include/linux/sched.h | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 4433afbf30a59..ef6fe7558cd8b 100755 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1216,9 +1216,6 @@ struct ravg { u64 mark_start; u32 sum, demand; u32 sum_history[RAVG_HIST_SIZE_MAX]; -#ifdef VENDOR_EDIT - unsigned mitigated:1; -#endif #ifdef CONFIG_SCHED_FREQ_INPUT u32 curr_window, prev_window; u16 active_windows; @@ -1344,11 +1341,7 @@ struct task_struct { atomic_t usage; unsigned int flags; /* per process flags, defined below */ unsigned int ptrace; - #ifdef VENDOR_EDIT - //huruihuan add for kill task in D status - unsigned int kill_flag; - struct timespec ttu; - #endif + #ifdef CONFIG_SMP struct llist_node wake_entry; int on_cpu; From 546930d7cc2e4388d55f88c515237e6faee6708b Mon Sep 17 00:00:00 2001 From: "www.anuprita804@gmail.com" Date: Fri, 18 Aug 2017 18:20:09 +0200 Subject: [PATCH 58/86] defconfig: disable SYSVIPC otherwise build fails with an error --- arch/arm64/configs/oneplus2_defconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/configs/oneplus2_defconfig b/arch/arm64/configs/oneplus2_defconfig index dadd570e4dce8..531693a85a4de 100644 --- a/arch/arm64/configs/oneplus2_defconfig +++ b/arch/arm64/configs/oneplus2_defconfig @@ -1,5 +1,5 @@ CONFIG_LOCALVERSION="-perf" -CONFIG_SYSVIPC=y +CONFIG_SYSVIPC=n CONFIG_SCHED_TUNE=y CONFIG_CGROUP_SCHEDTUNE=y CONFIG_CPU_FREQ_GOV_SCHED=y From 63783897277c472ecfa33d1193e27f42bf60555e Mon Sep 17 00:00:00 2001 From: Theodore Ts'o Date: Thu, 22 Jan 2015 12:10:43 -0500 Subject: [PATCH 59/86] percpu: add raw_cpu_ptr() for 3.18 ext4 backport Change-Id: Id2ca3694d1b75bc4f0ff3406dee4316275160a7d Signed-off-by: Theodore Ts'o Signed-off-by: Joe Maples --- include/asm-generic/percpu.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h index d17784ea37ff3..344c19565813c 100644 --- a/include/asm-generic/percpu.h +++ b/include/asm-generic/percpu.h @@ -87,6 +87,9 @@ extern void setup_per_cpu_areas(void); #endif /* SMP */ +/* 3.18 backport */ +#define raw_cpu_ptr(ptr) __this_cpu_ptr(ptr) + #ifndef PER_CPU_BASE_SECTION #ifdef CONFIG_SMP #define PER_CPU_BASE_SECTION ".data..percpu" From 11f89f1b508e9464cd3abe996262d37325158ab7 Mon Sep 17 00:00:00 2001 From: Giulio Cervera Date: Sun, 14 Feb 2016 11:46:09 +0100 Subject: [PATCH 60/86] msm8994: eas: disable debug message excessive log trigger watchdog Signed-off-by: mydongistiny Signed-off-by: Joe Maples --- kernel/sched/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 493ded52b5581..f35a7bdb177e2 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -8810,8 +8810,8 @@ sched_set_cpu_cstate(int cpu, int cstate, struct cpuidle_state *cpuidle, (cstate == 3 && cpu >= 4 && wakeup_energy == 155775 && wakeup_latency == 475) || (cstate == 4 && cpu < 4 && wakeup_energy == 46400 && wakeup_latency == 480) || (cstate == 4 && cpu >= 4 && wakeup_energy == 192156 && wakeup_latency == 636)) { - } else { - pr_info("TJK: CPU%d ->cstate=%d nrg=%d latency=%d\n", cpu, cstate, wakeup_energy, wakeup_latency); +// } else { +// pr_info("TJK: CPU%d ->cstate=%d nrg=%d latency=%d\n", cpu, cstate, wakeup_energy, wakeup_latency); } } From 5574f43cd2b3b90cc81b77f1530e25f24ab044ca Mon Sep 17 00:00:00 2001 From: Chris Redpath Date: Mon, 27 Mar 2017 15:00:14 +0100 Subject: [PATCH 61/86] EXPERIMENTAL: sched/fair: kick nohz idle balance for misfit task If there have misfit task on one CPU, current code does not handle this situation for nohz idle balance. As result, we can see the misfit task stays run on little core for long time. So this patch check if the CPU has misfit task or not. If has misfit task then kick nohz idle balance so finally can execute active balance. Change-Id: I117d3b7404296f8de11cb960a87a6b9a54a9f348 Signed-off-by: Leo Yan [taken from https://lists.linaro.org/pipermail/eas-dev/2016-September/000551.html] Signed-off-by: Chris Redpath Signed-off-by: mydongistiny Signed-off-by: Joe Maples --- kernel/sched/fair.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 5dc6e46f041bb..3df83f31a21a3 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8709,6 +8709,10 @@ static inline bool nohz_kick_needed(struct rq *rq) (!energy_aware() || cpu_overutilized(cpu))) return true; + /* Do idle load balance if there have misfit task */ + if (energy_aware() && rq->misfit_task) + return true; + rcu_read_lock(); sd = rcu_dereference(per_cpu(sd_busy, cpu)); if (sd && !energy_aware()) { From a8b36021925715f951d520b600420b9e0f6b49f3 Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Thu, 30 Jul 2015 16:53:30 +0100 Subject: [PATCH 62/86] BACKPORT: sched: EAS & cpu hotplug interoperability For Energy-Aware Scheduling (EAS) to work properly, even in the case that cpus are hot-plugged out, the energy model (EM) data on all energy-aware sched domains has to be present for all online cpus. Mainline sd hierarchy setup code will remove sd's which are not useful for task scheduling e.g. in the following situations: 1. Only one cpu remains in one cluster of a two cluster system. This remaining cpu only has DIE and no MC sd. 2. A complete cluster in a two-cluster system is hot-plugged out. The cpus of the remaining cluster only have MC and no DIE sd. To make sure that all online cpus keep all their energy-aware sd's, the sd degenerate functionality has been changed to not free sd's if their first sg contains EM data in case: 1. There is only one cpu left in the sd. 2. There have to be at least 2 sg's if certain sd flags are set. Instead of freeing such an sd it now clears only its SD_LOAD_BALANCE flag. This will make sure that the EAS functionality will always see all energy-aware sd's for all online cpus. It will introduce a (small ?) performance degradation since the hot-path macro for_each_domain() has to deal with sd's not contributing to task-scheduling at all now. There is the handling of newidle decay values before the SD_LOAD_BALANCE check in rebalance_domains(). But generally, code to make sure that task scheduling is not invoked on these sd's is in place (if (!(sd->flags & SD_LOAD_BALANCE)) already. This patch has been tested on a single (a7) cluster TC2 system. I.e. we could abandon the SYS sd level patch and use this patch to solve all problems related to sd topology setups and runtime changes. This patch should not be squashed. Test (w/ CONFIG_SCHED_DEBUG): JUNO: $ cat /proc/cpuinfo | grep "^CPU part" CPU part : 0xd03 CPU part : 0xd07 CPU part : 0xd07 CPU part : 0xd03 CPU part : 0xd03 CPU part : 0xd03 $ cat /proc/sys/kernel/sched_domain/cpu*/domain*/{name,flags} MC DIE MC DIE MC DIE MC DIE MC DIE MC DIE 33583 - 0x832f 4143 - 0x102f 33583 4143 33583 4143 33583 4143 33583 4143 33583 4143 Hotplug-out A57 cluster $ echo 0 > /sys/devices/system/cpu/cpu1/online $ echo 0 > /sys/devices/system/cpu/cpu2/online $ cat /proc/sys/kernel/sched_domain/cpu*/domain*/{name,flags} MC DIE MC DIE MC DIE MC DIE 33583 4142 - 0x102e <-- !SD_LOAD_BALANCE 33583 4142 33583 4142 33583 4142 Signed-off-by: Dietmar Eggemann Signed-off-by: mydongistiny Signed-off-by: Joe Maples --- kernel/sched/core.c | 18 +++++++++++------- kernel/sched/fair.c | 7 +++++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f35a7bdb177e2..dd5d2281dd210 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5557,9 +5557,6 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, if (!(sd->flags & SD_LOAD_BALANCE)) { printk("does not load-balance\n"); - if (sd->parent) - printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain" - " has parent"); return -1; } @@ -5652,8 +5649,12 @@ static inline bool sched_debug(void) static int sd_degenerate(struct sched_domain *sd) { - if (cpumask_weight(sched_domain_span(sd)) == 1) - return 1; + if (cpumask_weight(sched_domain_span(sd)) == 1) { + if (sd->groups->sge) + sd->flags &= ~SD_LOAD_BALANCE; + else + return 1; + } /* Following flags need at least 2 groups */ if (sd->flags & (SD_LOAD_BALANCE | @@ -5697,6 +5698,11 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) SD_PREFER_SIBLING | SD_SHARE_POWERDOMAIN | SD_SHARE_CAP_STATES); + if (parent->groups->sge) { + parent->flags &= ~SD_LOAD_BALANCE; + return 0; + } + if (nr_node_ids == 1) pflags &= ~SD_SERIALIZE; } @@ -6932,8 +6938,6 @@ static int build_sched_domains(const struct cpumask *cpu_map, *per_cpu_ptr(d.sd, i) = sd; if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP)) sd->flags |= SD_OVERLAP; - if (cpumask_equal(cpu_map, sched_domain_span(sd))) - break; } } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 3df83f31a21a3..e2eb39423ecd7 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7259,6 +7259,9 @@ static inline enum fbq_type fbq_classify_rq(struct rq *rq) } #endif /* CONFIG_NUMA_BALANCING */ +#define lb_sd_parent(sd) \ + (sd->parent && sd->parent->groups != sd->parent->groups->next) + /** * update_sd_lb_stats - Update sched_domain's statistics for load balancing. * @env: The load balancing environment. @@ -7341,7 +7344,7 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd env->src_grp_nr_running = sds->busiest_stat.sum_nr_running; - if (!env->sd->parent) { + if (!lb_sd_parent(env->sd)) { /* update overload indicator if we are at root domain */ if (env->dst_rq->rd->overload != overload) env->dst_rq->rd->overload = overload; @@ -7842,7 +7845,7 @@ static int load_balance(int this_cpu, struct rq *this_rq, int *continue_balancing) { int ld_moved, cur_ld_moved, active_balance = 0; - struct sched_domain *sd_parent = sd->parent; + struct sched_domain *sd_parent = lb_sd_parent(sd) ? sd->parent : NULL; struct sched_group *group; struct rq *busiest; unsigned long flags; From 50785716eb75e6269b042511b9f0ce2b3014bbb5 Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Fri, 25 Sep 2015 17:15:11 +0100 Subject: [PATCH 63/86] WIP: arm64: Enable frequency invariant scheduler load-tracking support Defines arch_scale_freq_capacity() to use cpufreq implementation. Including in topology.h like for the arm arch doesn't work because of CONFIG_COMPAT=y (Kernel support for 32-bit EL0). That's why cpufreq_scale_freq_capacity() has to be declared extern in topology.h. BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. Change-Id: I492c1851063220336b4ef6834303f2d44a6daf7a Signed-off-by: Dietmar Eggemann Signed-off-by: Joe Maples --- arch/arm64/include/asm/topology.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index abf8c9a8c9378..287c37ec78d50 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -24,9 +24,11 @@ void init_cpu_topology(void); void store_cpu_topology(unsigned int cpuid); const struct cpumask *cpu_coregroup_mask(int cpu); +#ifdef CONFIG_CPU_FREQ +#define arch_scale_freq_capacity cpufreq_scale_freq_capacity struct sched_domain; -extern unsigned long arch_scale_freq_capacity(struct sched_domain *sd, int cpu); -DECLARE_PER_CPU(atomic_long_t, cpu_freq_capacity); +extern unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu); +#endif #else From 4c7627d0afb1e2f1f51eae180f276f80baa65056 Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Wed, 23 Sep 2015 17:59:55 +0100 Subject: [PATCH 64/86] WIP: arm: Enable max freq invariant scheduler load-tracking and capacity support Maximum Frequency Invariance has to be part of Cpu Invariance because Frequency Invariance deals only with differences in load-tracking introduces by Dynamic Frequency Scaling and not with limiting the possible range of cpu frequency. By placing Maximum Frequency Invariance into Cpu Invariance, load-tracking is scaled via arch_scale_cpu_capacity() in __update_load_avg() and cpu capacity is scaled via arch_scale_cpu_capacity() in update_cpu_capacity(). To be able to save the extra multiplication in the scheduler hotpath (__update_load_avg()) we could: 1 Inform cpufreq about base cpu capacity at boot and let it handle scale_cpu_capacity() as well. 2 Use the cpufreq policy callback which would update a per-cpu current cpu_scale and this value would be return in scale_cpu_capacity(). 3 Use per-cpu current max_freq_scale and current cpu_scale with the current patch. BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. Change-Id: If5e9e0ba8ff5a5d3236b373dbce8c72ea71b5e18 Signed-off-by: Dietmar Eggemann Signed-off-by: Joe Maples --- arch/arm/kernel/topology.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm/kernel/topology.c b/arch/arm/kernel/topology.c index 68e332f9a05b9..cb55f2dc5fa21 100644 --- a/arch/arm/kernel/topology.c +++ b/arch/arm/kernel/topology.c @@ -44,7 +44,13 @@ static DEFINE_PER_CPU(unsigned long, cpu_scale); unsigned long arch_scale_freq_power(struct sched_domain *sd, int cpu) { +#if CONFIG_CPU_FREQ + unsigned long max_freq_scale = cpufreq_scale_max_freq_capacity(cpu); + + return per_cpu(cpu_scale, cpu) * max_freq_scale >> SCHED_CAPACITY_SHIFT; +#else return per_cpu(cpu_scale, cpu); +#endif } static void set_power_scale(unsigned int cpu, unsigned long power) From 47da84767975ab5f71db85482fc0875ab378792e Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Wed, 13 Jan 2016 02:44:40 -0800 Subject: [PATCH 65/86] CHROMIUM: cpufreq: Declare acct_update_power as static inline if needed If CONFIG_CPU_FREQ_STAT is not configured, builds fail with kernel/built-in.o: In function `account_user_time': kernel/sched/cputime.c:155: undefined reference to `acct_update_power' kernel/sched/cputime.c:155: undefined reference to `acct_update_power' kernel/built-in.o: In function `__account_system_time': kernel/sched/cputime.c:208: undefined reference to `acct_update_power' kernel/sched/cputime.c:208: undefined reference to `acct_update_power' Declare acct_update_power() as static inline if CONFIG_CPU_FREQ_STAT is not configured. Fixes: bf550d0b70dc ("ANDROID: sched: cpufreq: Update cpu_power field in the task_struct") BUG=chromium:583462 TEST=None (build) Change-Id: I64f5d0087504034a239014c31ee9cb9a12ad8611 Signed-off-by: Guenter Roeck Signed-off-by: Olof Johansson Reviewed-on: https://chromium-review.googlesource.com/325374 Commit-Ready: Daniel Kurtz Reviewed-by: Dylan Reid Signed-off-by: Joe Maples --- include/linux/cpufreq.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 78799917d052e..d4249bbe7b5a6 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -509,6 +509,16 @@ static inline int cpufreq_generic_exit(struct cpufreq_policy *policy) * CPUFREQ STATS * *********************************************************************/ +#ifdef CONFIG_CPU_FREQ_STAT + void acct_update_power(struct task_struct *p, cputime_t cputime); +#else + +static inline void acct_update_power(struct task_struct *p, cputime_t cputime) +{ +} + +#endif + #endif /* _LINUX_CPUFREQ_H */ From 5964dc5ec84501ad0cfb5be1f65beaf0f448d95f Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Fri, 10 Jul 2015 13:57:19 +0100 Subject: [PATCH 66/86] WIP: arm: Cpu invariant scheduler load-tracking and capacity support Provides the scheduler with a cpu scaling correction factor for more accurate load-tracking and cpu capacity handling. The Energy Model (EM) (in fact the capacity value of the last element of the capacity states vector of the core (MC) level sched_group_energy structure) is used instead of the arm arch specific cpu_efficiency and dtb property 'clock-frequency' values as the source for this cpu scaling factor. The cpu capacity value depends on the micro-architecture and the maximum frequency of the cpu. The maximum frequency part should not be confused with the frequency invariant scheduler load-tracking support which deals with frequency related scaling due to DFVS functionality. BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. Change-Id: I7588caa23bf3603eeb42223258fcc7e31182de2a Signed-off-by: Juri Lelli Signed-off-by: Dietmar Eggemann Signed-off-by: Joe Maples --- arch/arm/kernel/topology.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/arch/arm/kernel/topology.c b/arch/arm/kernel/topology.c index cb55f2dc5fa21..617846134a000 100644 --- a/arch/arm/kernel/topology.c +++ b/arch/arm/kernel/topology.c @@ -180,6 +180,8 @@ static void __init parse_dt_topology(void) } +static const struct sched_group_energy * const cpu_core_energy(int cpu); + /* * Look for a customed capacity of a CPU in the cpu_capacity table during the * boot. The update of all CPUs is in O(n^2) for heteregeneous system but the @@ -187,21 +189,14 @@ static void __init parse_dt_topology(void) */ void update_cpu_power(unsigned int cpu, unsigned long hwid) { - unsigned int idx = 0; - - /* look for the cpu's hwid in the cpu capacity table */ - for (idx = 0; idx < num_possible_cpus(); idx++) { - if (cpu_capacity[idx].hwid == hwid) - break; + unsigned long capacity = SCHED_CAPACITY_SCALE; - if (cpu_capacity[idx].hwid == -1) - return; + if (cpu_core_energy(cpu)) { + int max_cap_idx = cpu_core_energy(cpu)->nr_cap_states - 1; + capacity = cpu_core_energy(cpu)->cap_states[max_cap_idx].cap; } - if (idx == num_possible_cpus()) - return; - - set_power_scale(cpu, cpu_capacity[idx].capacity / middle_capacity); + set_capacity_scale(cpu, capacity); printk(KERN_INFO "CPU%u: update cpu_power %lu\n", cpu, arch_scale_freq_power(NULL, cpu)); From 2d7533e96fec7b08dc31bdb134994382d0eec7b0 Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Wed, 23 Sep 2015 12:47:48 +0100 Subject: [PATCH 67/86] WIP: arm: Enable frequency invariant scheduler load-tracking support Defines arch_scale_freq_capacity() to use cpufreq implementation. BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. Change-Id: I7b82e9935533aaf2f24903bb05e7f0b1bcdb6a88 Signed-off-by: Dietmar Eggemann Signed-off-by: Joe Maples --- arch/arm/include/asm/topology.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm/include/asm/topology.h b/arch/arm/include/asm/topology.h index 58b8b84adcd2c..bcefcfdca871b 100644 --- a/arch/arm/include/asm/topology.h +++ b/arch/arm/include/asm/topology.h @@ -27,6 +27,11 @@ void init_cpu_topology(void); void store_cpu_topology(unsigned int cpuid); const struct cpumask *cpu_coregroup_mask(int cpu); +#ifdef CONFIG_CPU_FREQ +#include +#define arch_scale_freq_capacity cpufreq_scale_freq_capacity +#endif + #else static inline void init_cpu_topology(void) { } From 9e3dbab87574bca0bd293f964ed44237d48e7e7c Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Fri, 25 Sep 2015 17:34:15 +0100 Subject: [PATCH 68/86] WIP: arm64: Enable max freq invariant scheduler load-tracking and capacity support Maximum Frequency Invariance has to be part of Cpu Invariance because Frequency Invariance deals only with differences in load-tracking introduces by Dynamic Frequency Scaling and not with limiting the possible range of cpu frequency. By placing Maximum Frequency Invariance into Cpu Invariance, load-tracking is scaled via arch_scale_cpu_capacity() in __update_load_avg() and cpu capacity is scaled via arch_scale_cpu_capacity() in update_cpu_capacity(). To be able to save the extra multiplication in the scheduler hotpath (__update_load_avg()) we could: 1 Inform cpufreq about base cpu capacity at boot and let it handle scale_cpu_capacity() as well. 2 Use the cpufreq policy callback which would update a per-cpu current cpu_scale and this value would be return in scale_cpu_capacity(). 3 Use per-cpu current max_freq_scale and current cpu_scale with the current patch. Including in topology.h like for the arm arch doesn't work because of CONFIG_COMPAT=y (Kernel support for 32-bit EL0). That's why cpufreq_scale_max_freq_capacity() has to be declared extern in topology.h. BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. Change-Id: I2c25fbc7748865651975ba212e1d9b2976d8f7d6 Signed-off-by: Dietmar Eggemann Signed-off-by: Joe Maples --- arch/arm64/include/asm/topology.h | 1 + arch/arm64/kernel/topology.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index 287c37ec78d50..8f384eff636c2 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -28,6 +28,7 @@ const struct cpumask *cpu_coregroup_mask(int cpu); #define arch_scale_freq_capacity cpufreq_scale_freq_capacity struct sched_domain; extern unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu); +extern unsigned long cpufreq_scale_max_freq_capacity(int cpu); #endif #else diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 0de1aa7b28cb2..41f4bd02fd022 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -41,7 +41,13 @@ static DEFINE_PER_CPU(unsigned long, cpu_scale); unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu) { +#ifdef CONFIG_CPU_FREQ + unsigned long max_freq_scale = cpufreq_scale_max_freq_capacity(cpu); + + return per_cpu(cpu_scale, cpu) * max_freq_scale >> SCHED_CAPACITY_SHIFT; +#else return per_cpu(cpu_scale, cpu); +#endif } static void set_capacity_scale(unsigned int cpu, unsigned long capacity) From 1e02bcd4ae27e3606271d8489333414dc61f8922 Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Thu, 17 Sep 2015 16:10:56 +0100 Subject: [PATCH 69/86] WIP: cpufreq: Frequency invariant scheduler load-tracking support Implements cpufreq_scale_freq_capacity() to provide the scheduler with a frequency scaling correction factor for more accurate load-tracking. The factor is: current_freq(cpu) << SCHED_CAPACITY_SHIFT / max_freq(cpu) In fact, freq_scale should be a struct cpufreq_policy data member. But this would require that the scheduler hot path (__update_load_avg()) would have to grab the cpufreq lock. This can be avoided by using per-cpu data initialized to SCHED_CAPACITY_SCALE for freq_scale. BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. Change-Id: I8aa1c424a33794db78c35a80c57184d992e71d2b Signed-off-by: Dietmar Eggemann Signed-off-by: Joe Maples --- drivers/cpufreq/cpufreq.c | 28 +++++++++++++++++++++++++++- include/linux/cpufreq.h | 2 ++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 51e469d900fb4..5918e7617acfd 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -281,6 +281,31 @@ static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci) } #endif +/********************************************************************* + * FREQUENCY INVARIANT CPU CAPACITY * + *********************************************************************/ + +static DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE; + +static void +scale_freq_capacity(struct cpufreq_policy *policy, struct cpufreq_freqs *freqs) +{ + unsigned long cur = freqs ? freqs->new : policy->cur; + unsigned long scale = (cur << SCHED_CAPACITY_SHIFT) / policy->max; + int cpu; + + pr_debug("cpus %*pbl cur/cur max freq %lu/%u kHz freq scale %lu\n", + cpumask_pr_args(policy->cpus), cur, policy->max, scale); + + for_each_cpu(cpu, policy->cpus) + per_cpu(freq_scale, cpu) = scale; +} + +unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu) +{ + return per_cpu(freq_scale, cpu); +} + static void __cpufreq_notify_transition(struct cpufreq_policy *policy, struct cpufreq_freqs *freqs, unsigned int state) { @@ -343,7 +368,6 @@ void cpufreq_notify_transition(struct cpufreq_policy *policy, } EXPORT_SYMBOL_GPL(cpufreq_notify_transition); - /********************************************************************* * SYSFS INTERFACE * *********************************************************************/ @@ -2042,6 +2066,8 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_NOTIFY, new_policy); + scale_freq_capacity(new_policy, NULL); + policy->min = new_policy->min; policy->max = new_policy->max; trace_cpu_frequency_limits(policy->max, policy->min, policy->cpu); diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index d4249bbe7b5a6..6497bc7f8cd73 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -521,4 +521,6 @@ static inline void acct_update_power(struct task_struct *p, cputime_t cputime) #endif +struct sched_domain; +unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu); #endif /* _LINUX_CPUFREQ_H */ From 110e9e8851de6c44a61c6317e0ac8c1d2215aad6 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 13 Feb 2015 14:36:57 -0800 Subject: [PATCH 70/86] UPSTREAM: cpumask, nodemask: implement cpumask/nodemask_pr_args() printf family of functions can now format bitmaps using '%*pb[l]' and all cpumask and nodemask formatting will be converted to use it. To ease printing these masks with '%*pb[l]' which require two params - the number of bits and the actual bitmap, this patch implement cpumask_pr_args() and nodemask_pr_args() which can be used to provide arguments for '%*pb[l]' Signed-off-by: Tejun Heo Cc: Rusty Russell Cc: "David S. Miller" Cc: "James E.J. Bottomley" Cc: "John W. Linville" Cc: "Paul E. McKenney" Cc: Benjamin Herrenschmidt Cc: Chris Metcalf Cc: Chris Zankel Cc: Christoph Lameter Cc: Dmitry Torokhov Cc: Fenghua Yu Cc: Greg Kroah-Hartman Cc: Ingo Molnar Cc: Li Zefan Cc: Max Filippov Cc: Mike Travis Cc: Pekka Enberg Cc: Peter Zijlstra Cc: Russell King Cc: Steffen Klassert Cc: Steven Rostedt Cc: Thomas Gleixner Cc: Tony Luck Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit f1bbc032e45106400905ebb47550983af4690b0b) Signed-off-by: Nicolas Boichat BUG=chromium:474744 TEST=x86-64 allmodconfig can compile arch/x86/mm/numa.o Change-Id: I91286b7bec2ae3c4b3660828e344edf2173f08e4 Reviewed-on: https://chromium-review.googlesource.com/300833 Commit-Ready: Nicolas Boichat Tested-by: Nicolas Boichat Reviewed-by: Benson Leung Reviewed-by: Daniel Kurtz Signed-off-by: Joe Maples --- include/linux/cpumask.h | 8 ++++++++ include/linux/nodemask.h | 17 +++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index d08e4d2a9b925..7abda3b903db5 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -22,6 +22,14 @@ typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t; */ #define cpumask_bits(maskp) ((maskp)->bits) +/** + * cpumask_pr_args - printf args to output a cpumask + * @maskp: cpumask to be printed + * + * Can be used to provide arguments for '%*pb[l]' when printing a cpumask. + */ +#define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp) + #if NR_CPUS == 1 #define nr_cpu_ids 1 #else diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h index 4e2cbfa640b72..51f6783a2fef9 100644 --- a/include/linux/nodemask.h +++ b/include/linux/nodemask.h @@ -98,6 +98,23 @@ typedef struct { DECLARE_BITMAP(bits, MAX_NUMNODES); } nodemask_t; extern nodemask_t _unused_nodemask_arg_; +/** + * nodemask_pr_args - printf args to output a nodemask + * @maskp: nodemask to be printed + * + * Can be used to provide arguments for '%*pb[l]' when printing a nodemask. + */ +#define nodemask_pr_args(maskp) MAX_NUMNODES, (maskp)->bits + +/* + * The inline keyword gives the compiler room to decide to inline, or + * not inline a function as it sees best. However, as these functions + * are called in both __init and non-__init functions, if they are not + * inlined we will end up with a section mis-match error (of the type of + * freeable items not being freed). So we must use __always_inline here + * to fix the problem. If other functions in the future also end up in + * this situation they will also need to be annotated as __always_inline + */ #define node_set(node, dst) __node_set((node), &(dst)) static inline void __node_set(int node, volatile nodemask_t *dstp) { From d0a30eed1437fbf87011d5f1fb6d198024abdc15 Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Tue, 22 Sep 2015 16:47:48 +0100 Subject: [PATCH 71/86] WIP: cpufreq: Max freq invariant scheduler load-tracking and cpu capacity support Implements cpufreq_scale_max_freq_capacity() to provide the scheduler with a maximum frequency scaling correction factor for more accurate load-tracking and cpu capacity handling by being able to deal with frequency capping. This scaling factor describes the influence of running a cpu with a current maximum frequency lower than the absolute possible maximum frequency on load tracking and cpu capacity. The factor is: current_max_freq(cpu) << SCHED_CAPACITY_SHIFT / max_freq(cpu) In fact, max_freq_scale should be a struct cpufreq_policy data member. But this would require that the scheduler hot path (__update_load_avg()) would have to grab the cpufreq lock. This can be avoided by using per-cpu data initialized to SCHED_CAPACITY_SCALE for max_freq_scale. BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. Change-Id: I5bcf45a208deecd5d8d063adb9e1f06bda44ce98 Signed-off-by: Dietmar Eggemann Signed-off-by: Joe Maples --- drivers/cpufreq/cpufreq.c | 19 +++++++++++++++++++ include/linux/cpufreq.h | 1 + 2 files changed, 20 insertions(+) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 5918e7617acfd..24def50615b76 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -286,12 +286,14 @@ static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci) *********************************************************************/ static DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE; +static DEFINE_PER_CPU(unsigned long, max_freq_scale) = SCHED_CAPACITY_SCALE; static void scale_freq_capacity(struct cpufreq_policy *policy, struct cpufreq_freqs *freqs) { unsigned long cur = freqs ? freqs->new : policy->cur; unsigned long scale = (cur << SCHED_CAPACITY_SHIFT) / policy->max; + struct cpufreq_cpuinfo *cpuinfo = &policy->cpuinfo; int cpu; pr_debug("cpus %*pbl cur/cur max freq %lu/%u kHz freq scale %lu\n", @@ -299,6 +301,18 @@ scale_freq_capacity(struct cpufreq_policy *policy, struct cpufreq_freqs *freqs) for_each_cpu(cpu, policy->cpus) per_cpu(freq_scale, cpu) = scale; + + if (freqs) + return; + + scale = (policy->max << SCHED_CAPACITY_SHIFT) / cpuinfo->max_freq; + + pr_debug("cpus %*pbl cur max/max freq %u/%u kHz max freq scale %lu\n", + cpumask_pr_args(policy->cpus), policy->max, cpuinfo->max_freq, + scale); + + for_each_cpu(cpu, policy->cpus) + per_cpu(max_freq_scale, cpu) = scale; } unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu) @@ -306,6 +320,11 @@ unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu) return per_cpu(freq_scale, cpu); } +unsigned long cpufreq_scale_max_freq_capacity(int cpu) +{ + return per_cpu(max_freq_scale, cpu); +} + static void __cpufreq_notify_transition(struct cpufreq_policy *policy, struct cpufreq_freqs *freqs, unsigned int state) { diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 6497bc7f8cd73..70ee2c9e2e690 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -523,4 +523,5 @@ static inline void acct_update_power(struct task_struct *p, cputime_t cputime) struct sched_domain; unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu); +unsigned long cpufreq_scale_max_freq_capacity(int cpu); #endif /* _LINUX_CPUFREQ_H */ From 82f32308337808252d62491e6a51a1325fb257d3 Mon Sep 17 00:00:00 2001 From: Morten Rasmussen Date: Tue, 13 Jan 2015 13:43:28 +0000 Subject: [PATCH 72/86] WIP: sched: Documentation for scheduler energy cost model This documentation patch provides an overview of the experimental scheduler energy costing model, associated data structures, and a reference recipe on how platforms can be characterized to derive energy models. BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. Change-Id: I660906d9e23f79e2e0c4a0459c63ade62c61c4fa Signed-off-by: Morten Rasmussen Signed-off-by: Joe Maples --- Documentation/scheduler/sched-energy.txt | 362 +++++++++++++++++++++++ 1 file changed, 362 insertions(+) create mode 100644 Documentation/scheduler/sched-energy.txt diff --git a/Documentation/scheduler/sched-energy.txt b/Documentation/scheduler/sched-energy.txt new file mode 100644 index 0000000000000..dab2f9088b336 --- /dev/null +++ b/Documentation/scheduler/sched-energy.txt @@ -0,0 +1,362 @@ +Energy cost model for energy-aware scheduling (EXPERIMENTAL) + +Introduction +============= + +The basic energy model uses platform energy data stored in sched_group_energy +data structures attached to the sched_groups in the sched_domain hierarchy. The +energy cost model offers two functions that can be used to guide scheduling +decisions: + +1. static unsigned int sched_group_energy(struct energy_env *eenv) +2. static int energy_diff(struct energy_env *eenv) + +sched_group_energy() estimates the energy consumed by all cpus in a specific +sched_group including any shared resources owned exclusively by this group of +cpus. Resources shared with other cpus are excluded (e.g. later level caches). + +energy_diff() estimates the total energy impact of a utilization change. That +is, adding, removing, or migrating utilization (tasks). + +Both functions use a struct energy_env to specify the scenario to be evaluated: + + struct energy_env { + struct sched_group *sg_top; + struct sched_group *sg_cap; + int cap_idx; + int util_delta; + int src_cpu; + int dst_cpu; + int energy; + }; + +sg_top: sched_group to be evaluated. Not used by energy_diff(). + +sg_cap: sched_group covering the cpus in the same frequency domain. Set by +sched_group_energy(). + +cap_idx: Capacity state to be used for energy calculations. Set by +find_new_capacity(). + +util_delta: Amount of utilization to be added, removed, or migrated. + +src_cpu: Source cpu from where 'util_delta' utilization is removed. Should be +-1 if no source (e.g. task wake-up). + +dst_cpu: Destination cpu where 'util_delta' utilization is added. Should be -1 +if utilization is removed (e.g. terminating tasks). + +energy: Result of sched_group_energy(). + +The metric used to represent utilization is the actual per-entity running time +averaged over time using a geometric series. Very similar to the existing +per-entity load-tracking, but _not_ scaled by task priority and capped by the +capacity of the cpu. The latter property does mean that utilization may +underestimate the compute requirements for task on fully/over utilized cpus. +The greatest potential for energy savings without affecting performance too much +is scenarios where the system isn't fully utilized. If the system is deemed +fully utilized load-balancing should be done with task load (includes task +priority) instead in the interest of fairness and performance. + + +Background and Terminology +=========================== + +To make it clear from the start: + +energy = [joule] (resource like a battery on powered devices) +power = energy/time = [joule/second] = [watt] + +The goal of energy-aware scheduling is to minimize energy, while still getting +the job done. That is, we want to maximize: + + performance [inst/s] + -------------------- + power [W] + +which is equivalent to minimizing: + + energy [J] + ----------- + instruction + +while still getting 'good' performance. It is essentially an alternative +optimization objective to the current performance-only objective for the +scheduler. This alternative considers two objectives: energy-efficiency and +performance. Hence, there needs to be a user controllable knob to switch the +objective. Since it is early days, this is currently a sched_feature +(ENERGY_AWARE). + +The idea behind introducing an energy cost model is to allow the scheduler to +evaluate the implications of its decisions rather than applying energy-saving +techniques blindly that may only have positive effects on some platforms. At +the same time, the energy cost model must be as simple as possible to minimize +the scheduler latency impact. + +Platform topology +------------------ + +The system topology (cpus, caches, and NUMA information, not peripherals) is +represented in the scheduler by the sched_domain hierarchy which has +sched_groups attached at each level that covers one or more cpus (see +sched-domains.txt for more details). To add energy awareness to the scheduler +we need to consider power and frequency domains. + +Power domain: + +A power domain is a part of the system that can be powered on/off +independently. Power domains are typically organized in a hierarchy where you +may be able to power down just a cpu or a group of cpus along with any +associated resources (e.g. shared caches). Powering up a cpu means that all +power domains it is a part of in the hierarchy must be powered up. Hence, it is +more expensive to power up the first cpu that belongs to a higher level power +domain than powering up additional cpus in the same high level domain. Two +level power domain hierarchy example: + + Power source + +-------------------------------+----... +per group PD G G + | +----------+ | + +--------+-------| Shared | (other groups) +per-cpu PD G G | resource | + | | +----------+ + +-------+ +-------+ + | CPU 0 | | CPU 1 | + +-------+ +-------+ + +Frequency domain: + +Frequency domains (P-states) typically cover the same group of cpus as one of +the power domain levels. That is, there might be several smaller power domains +sharing the same frequency (P-state) or there might be a power domain spanning +multiple frequency domains. + +From a scheduling point of view there is no need to know the actual frequencies +[Hz]. All the scheduler cares about is the compute capacity available at the +current state (P-state) the cpu is in and any other available states. For that +reason, and to also factor in any cpu micro-architecture differences, compute +capacity scaling states are called 'capacity states' in this document. For SMP +systems this is equivalent to P-states. For mixed micro-architecture systems +(like ARM big.LITTLE) it is P-states scaled according to the micro-architecture +performance relative to the other cpus in the system. + +Energy modelling: +------------------ + +Due to the hierarchical nature of the power domains, the most obvious way to +model energy costs is therefore to associate power and energy costs with +domains (groups of cpus). Energy costs of shared resources are associated with +the group of cpus that share the resources, only the cost of powering the +cpu itself and any private resources (e.g. private L1 caches) is associated +with the per-cpu groups (lowest level). + +For example, for an SMP system with per-cpu power domains and a cluster level +(group of cpus) power domain we get the overall energy costs to be: + + energy = energy_cluster + n * energy_cpu + +where 'n' is the number of cpus powered up and energy_cluster is the cost paid +as soon as any cpu in the cluster is powered up. + +The power and frequency domains can naturally be mapped onto the existing +sched_domain hierarchy and sched_groups by adding the necessary data to the +existing data structures. + +The energy model considers energy consumption from two contributors (shown in +the illustration below): + +1. Busy energy: Energy consumed while a cpu and the higher level groups that it +belongs to are busy running tasks. Busy energy is associated with the state of +the cpu, not an event. The time the cpu spends in this state varies. Thus, the +most obvious platform parameter for this contribution is busy power +(energy/time). + +2. Idle energy: Energy consumed while a cpu and higher level groups that it +belongs to are idle (in a C-state). Like busy energy, idle energy is associated +with the state of the cpu. Thus, the platform parameter for this contribution +is idle power (energy/time). + +Energy consumed during transitions from an idle-state (C-state) to a busy state +(P-state) or going the other way is ignored by the model to simplify the energy +model calculations. + + + Power + ^ + | busy->idle idle->busy + | transition transition + | + | _ __ + | / \ / \__________________ + |______________/ \ / + | \ / + | Busy \ Idle / Busy + | low P-state \____________/ high P-state + | + +------------------------------------------------------------> time + +Busy |--------------| |-----------------| + +Wakeup |------| |------| + +Idle |------------| + + +The basic algorithm +==================== + +The basic idea is to determine the total energy impact when utilization is +added or removed by estimating the impact at each level in the sched_domain +hierarchy starting from the bottom (sched_group contains just a single cpu). +The energy cost comes from busy time (sched_group is awake because one or more +cpus are busy) and idle time (in an idle-state). Energy model numbers account +for energy costs associated with all cpus in the sched_group as a group. + + for_each_domain(cpu, sd) { + sg = sched_group_of(cpu) + energy_before = curr_util(sg) * busy_power(sg) + + (1-curr_util(sg)) * idle_power(sg) + energy_after = new_util(sg) * busy_power(sg) + + (1-new_util(sg)) * idle_power(sg) + energy_diff += energy_before - energy_after + + } + + return energy_diff + +{curr, new}_util: The cpu utilization at the lowest level and the overall +non-idle time for the entire group for higher levels. Utilization is in the +range 0.0 to 1.0 in the pseudo-code. + +busy_power: The power consumption of the sched_group. + +idle_power: The power consumption of the sched_group when idle. + +Note: It is a fundamental assumption that the utilization is (roughly) scale +invariant. Task utilization tracking factors in any frequency scaling and +performance scaling differences due to difference cpu microarchitectures such +that task utilization can be used across the entire system. + + +Platform energy data +===================== + +struct sched_group_energy can be attached to sched_groups in the sched_domain +hierarchy and has the following members: + +cap_states: + List of struct capacity_state representing the supported capacity states + (P-states). struct capacity_state has two members: cap and power, which + represents the compute capacity and the busy_power of the state. The + list must be ordered by capacity low->high. + +nr_cap_states: + Number of capacity states in cap_states list. + +idle_states: + List of struct idle_state containing idle_state power cost for each + idle-state supported by the system orderd by shallowest state first. + All states must be included at all level in the hierarchy, i.e. a + sched_group spanning just a single cpu must also include coupled + idle-states (cluster states). In addition to the cpuidle idle-states, + the list must also contain an entry for the idling using the arch + default idle (arch_idle_cpu()). Despite this state may not be a true + hardware idle-state it is considered the shallowest idle-state in the + energy model and must be the first entry. cpus may enter this state + (possibly 'active idling') if cpuidle decides not enter a cpuidle + idle-state. Default idle may not be used when cpuidle is enabled. + In this case, it should just be a copy of the first cpuidle idle-state. + +nr_idle_states: + Number of idle states in idle_states list. + +There are no unit requirements for the energy cost data. Data can be normalized +with any reference, however, the normalization must be consistent across all +energy cost data. That is, one bogo-joule/watt must be the same quantity for +data, but we don't care what it is. + +A recipe for platform characterization +======================================= + +Obtaining the actual model data for a particular platform requires some way of +measuring power/energy. There isn't a tool to help with this (yet). This +section provides a recipe for use as reference. It covers the steps used to +characterize the ARM TC2 development platform. This sort of measurements is +expected to be done anyway when tuning cpuidle and cpufreq for a given +platform. + +The energy model needs two types of data (struct sched_group_energy holds +these) for each sched_group where energy costs should be taken into account: + +1. Capacity state information + +A list containing the compute capacity and power consumption when fully +utilized attributed to the group as a whole for each available capacity state. +At the lowest level (group contains just a single cpu) this is the power of the +cpu alone without including power consumed by resources shared with other cpus. +It basically needs to fit the basic modelling approach described in "Background +and Terminology" section: + + energy_system = energy_shared + n * energy_cpu + +for a system containing 'n' busy cpus. Only 'energy_cpu' should be included at +the lowest level. 'energy_shared' is included at the next level which +represents the group of cpus among which the resources are shared. + +This model is, of course, a simplification of reality. Thus, power/energy +attributions might not always exactly represent how the hardware is designed. +Also, busy power is likely to depend on the workload. It is therefore +recommended to use a representative mix of workloads when characterizing the +capacity states. + +If the group has no capacity scaling support, the list will contain a single +state where power is the busy power attributed to the group. The capacity +should be set to a default value (1024). + +When frequency domains include multiple power domains, the group representing +the frequency domain and all child groups share capacity states. This must be +indicated by setting the SD_SHARE_CAP_STATES sched_domain flag. All groups at +all levels that share the capacity state must have the list of capacity states +with the power set to the contribution of the individual group. + +2. Idle power information + +Stored in the idle_states list. The power number is the group idle power +consumption in each idle state as well when the group is idle but has not +entered an idle-state ('active idle' as mentioned earlier). Due to the way the +energy model is defined, the idle power of the deepest group idle state can +alternatively be accounted for in the parent group busy power. In that case the +group idle state power values are offset such that the idle power of the +deepest state is zero. It is less intuitive, but it is easier to measure as +idle power consumed by the group and the busy/idle power of the parent group +cannot be distinguished without per group measurement points. + +Measuring capacity states and idle power: + +The capacity states' capacity and power can be estimated by running a benchmark +workload at each available capacity state. By restricting the benchmark to run +on subsets of cpus it is possible to extrapolate the power consumption of +shared resources. + +ARM TC2 has two clusters of two and three cpus respectively. Each cluster has a +shared L2 cache. TC2 has on-chip energy counters per cluster. Running a +benchmark workload on just one cpu in a cluster means that power is consumed in +the cluster (higher level group) and a single cpu (lowest level group). Adding +another benchmark task to another cpu increases the power consumption by the +amount consumed by the additional cpu. Hence, it is possible to extrapolate the +cluster busy power. + +For platforms that don't have energy counters or equivalent instrumentation +built-in, it may be possible to use an external DAQ to acquire similar data. + +If the benchmark includes some performance score (for example sysbench cpu +benchmark), this can be used to record the compute capacity. + +Measuring idle power requires insight into the idle state implementation on the +particular platform. Specifically, if the platform has coupled idle-states (or +package states). To measure non-coupled per-cpu idle-states it is necessary to +keep one cpu busy to keep any shared resources alive to isolate the idle power +of the cpu from idle/busy power of the shared resources. The cpu can be tricked +into different per-cpu idle states by disabling the other states. Based on +various combinations of measurements with specific cpus busy and disabling +idle-states it is possible to extrapolate the idle-state power. From 7b93be4963093a062c27dc7546c431e5ff07a0d7 Mon Sep 17 00:00:00 2001 From: Juri Lelli Date: Thu, 30 Apr 2015 11:53:48 +0100 Subject: [PATCH 73/86] arm64: Cpu invariant scheduler load-tracking and capacity support Provides the scheduler with a cpu scaling correction factor for more accurate load-tracking and cpu capacity handling. The Energy Model (EM) (in fact the capacity value of the last element of the capacity states vector of the core (MC) level sched_group_energy structure) is used as the source for this cpu scaling factor. The cpu capacity value depends on the micro-architecture and the maximum frequency of the cpu. The maximum frequency part should not be confused with the frequency invariant scheduler load-tracking support which deals with frequency related scaling due to DFVS functionality. BUG=chrome-os-partner:44828 TEST=Boot kernel on Oak. Change-Id: Iddc0f792184ad9d3ad6cb1cfc405ff6e3778d194 Signed-off-by: Juri Lelli Signed-off-by: Dietmar Eggemann [ @mydongistiny: updated for 3.10 with EAS ] Signed-off-by: mydongistiny [ @frap129: ported to ether kernel with EAS ] Signed-off-by: Joe Maples --- arch/arm64/include/asm/topology.h | 4 +- arch/arm64/kernel/topology.c | 186 +----------------------------- 2 files changed, 8 insertions(+), 182 deletions(-) diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index 8f384eff636c2..4ea11bad40025 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -24,12 +24,14 @@ void init_cpu_topology(void); void store_cpu_topology(unsigned int cpuid); const struct cpumask *cpu_coregroup_mask(int cpu); +struct sched_domain; #ifdef CONFIG_CPU_FREQ #define arch_scale_freq_capacity cpufreq_scale_freq_capacity -struct sched_domain; extern unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu); extern unsigned long cpufreq_scale_max_freq_capacity(int cpu); #endif +#define arch_scale_cpu_capacity scale_cpu_capacity +extern unsigned long scale_cpu_capacity(struct sched_domain *sd, int cpu); #else diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 41f4bd02fd022..dbee05e376626 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -26,28 +26,11 @@ #include #include -/* - * cpu power table - * This per cpu data structure describes the relative capacity of each core. - * On a heteregenous system, cores don't have the same computation capacity - * and we reflect that difference in the cpu_power field so the scheduler can - * take this difference into account during load balance. A per cpu structure - * is preferred because each CPU updates its own cpu_power field during the - * load balance except for idle cores. One idle core is selected to run the - * rebalance_domains for all idle cores and the cpu_power can be updated - * during this sequence. - */ -static DEFINE_PER_CPU(unsigned long, cpu_scale); +static DEFINE_PER_CPU(unsigned long, cpu_scale) = SCHED_CAPACITY_SCALE; -unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu) +unsigned long scale_cpu_capacity(struct sched_domain *sd, int cpu) { -#ifdef CONFIG_CPU_FREQ - unsigned long max_freq_scale = cpufreq_scale_max_freq_capacity(cpu); - - return per_cpu(cpu_scale, cpu) * max_freq_scale >> SCHED_CAPACITY_SHIFT; -#else return per_cpu(cpu_scale, cpu); -#endif } static void set_capacity_scale(unsigned int cpu, unsigned long capacity) @@ -193,46 +176,6 @@ static int __init parse_cluster(struct device_node *cluster, int depth) return 0; } -struct cpu_efficiency { - const char *compatible; - unsigned long efficiency; -}; - -/* - * Table of relative efficiency of each processors - * The efficiency value must fit in 20bit and the final - * cpu_scale value must be in the range - * 0 < cpu_scale < SCHED_CAPACITY_SCALE - */ -static const struct cpu_efficiency table_efficiency[] = { - { "arm,cortex-a72", 4186 }, - { "arm,cortex-a57", 3891 }, - { "arm,cortex-a53", 2048 }, - { NULL, }, -}; - -static unsigned long *__cpu_capacity; -#define cpu_capacity(cpu) __cpu_capacity[cpu] - -static unsigned long middle_capacity = 1; - -#ifdef TJK_HMP -static DEFINE_PER_CPU(unsigned long, cpu_efficiency) = SCHED_CAPACITY_SCALE; - -unsigned long arch_get_cpu_efficiency(int cpu) -{ - return per_cpu(cpu_efficiency, cpu); -} -#endif - -/* - * Iterate all CPUs' descriptor in DT and compute the efficiency - * (as per table_efficiency). Also calculate a middle efficiency - * as close as possible to (max{eff_i} - min{eff_i}) / 2 - * This is later used to scale the cpu_power field such that an - * 'average' CPU is of middle power. Also see the comments near - * table_efficiency[] and update_cpu_power(). - */ static int __init parse_dt_topology(void) { struct device_node *cn, *map; @@ -276,115 +219,6 @@ static int __init parse_dt_topology(void) return ret; } -/* - * Scheduler load-tracking scale-invariance - * - * Provides the scheduler with a scale-invariance correction factor that - * compensates for frequency scaling (arch_scale_freq_capacity()). The scaling - * factor is updated in smp.c - */ -unsigned long arch_scale_freq_capacity(struct sched_domain *sd, int cpu) -{ - unsigned long curr = atomic_long_read(&per_cpu(cpu_freq_capacity, cpu)); - - if (!curr) { - return SCHED_CAPACITY_SCALE; - } - - return curr; -} - -static void __init parse_dt_cpu_capacity(void) -{ - const struct cpu_efficiency *cpu_eff; - struct device_node *cn; - unsigned long min_capacity = ULONG_MAX; - unsigned long max_capacity = 0; - unsigned long capacity = 0; - int cpu; - - __cpu_capacity = kcalloc(nr_cpu_ids, sizeof(*__cpu_capacity), - GFP_NOWAIT); - - for_each_possible_cpu(cpu) { - const u32 *rate; - int len; - - /* Too early to use cpu->of_node */ - cn = of_get_cpu_node(cpu, NULL); - if (!cn) { - pr_err("Missing device node for CPU %d\n", cpu); - continue; - } - - for (cpu_eff = table_efficiency; cpu_eff->compatible; cpu_eff++) - if (of_device_is_compatible(cn, cpu_eff->compatible)) - break; - - if (cpu_eff->compatible == NULL) { - pr_warn("%s: Unknown CPU type\n", cn->full_name); - continue; - } - -#ifdef TJK_HMP - per_cpu(cpu_efficiency, cpu) = cpu_eff->efficiency; -#endif - - rate = of_get_property(cn, "clock-frequency", &len); - if (!rate || len != 4) { - pr_err("%s: Missing clock-frequency property\n", - cn->full_name); - continue; - } - - capacity = ((be32_to_cpup(rate)) >> 20) * cpu_eff->efficiency; - - /* Save min capacity of the system */ - if (capacity < min_capacity) - min_capacity = capacity; - - /* Save max capacity of the system */ - if (capacity > max_capacity) - max_capacity = capacity; - - cpu_capacity(cpu) = capacity; - } - - /* If min and max capacities are equal we bypass the update of the - * cpu_scale because all CPUs have the same capacity. Otherwise, we - * compute a middle_capacity factor that will ensure that the capacity - * of an 'average' CPU of the system will be as close as possible to - * SCHED_CAPACITY_SCALE, which is the default value, but with the - * constraint explained near table_efficiency[]. - */ - if (min_capacity == max_capacity) - return; - else if (4 * max_capacity < (3 * (max_capacity + min_capacity))) - middle_capacity = (min_capacity + max_capacity) - >> (SCHED_CAPACITY_SHIFT+1); - else - middle_capacity = ((max_capacity / 3) - >> (SCHED_CAPACITY_SHIFT-1)) + 1; -} - -#ifdef TJK_HMP -/* - * Look for a customed capacity of a CPU in the cpu_topo_data table during the - * boot. The update of all CPUs is in O(n^2) for heteregeneous system but the - * function returns directly for SMP system. - */ -static void update_cpu_power(unsigned int cpu) -{ - if (!cpu_capacity(cpu)) - return; - - set_power_scale(cpu, cpu_capacity(cpu) / middle_capacity); - - pr_info("CPU%u: update cpu_power %lu\n", - cpu, arch_scale_cpu_capacity(NULL, cpu)); -} -#endif - /* * cpu topology table */ @@ -410,16 +244,15 @@ const struct cpumask *cpu_coregroup_mask(int cpu) static void update_cpu_capacity(unsigned int cpu) { - unsigned long capacity; + unsigned long capacity = SCHED_CAPACITY_SCALE; - if (!cpu_core_energy(cpu)) { - capacity = SCHED_CAPACITY_SCALE; - } else { + if (cpu_core_energy(cpu)) { int max_cap_idx = cpu_core_energy(cpu)->nr_cap_states - 1; capacity = cpu_core_energy(cpu)->cap_states[max_cap_idx].cap; } set_capacity_scale(cpu, capacity); + pr_info("CPU%d: update cpu_capacity %lu\n", cpu, arch_scale_cpu_capacity(NULL, cpu)); } @@ -517,13 +350,6 @@ static void __init reset_cpu_topology(void) } } -static void __init reset_cpu_capacity(void) -{ - unsigned int cpu; - for_each_possible_cpu(cpu) - set_capacity_scale(cpu, SCHED_CAPACITY_SCALE); -} - static inline const struct sched_group_energy *cpu_cluster_energy(int cpu) { struct sched_group_energy *sge = sge_array[cpu][SD_LEVEL1]; @@ -563,7 +389,5 @@ void __init init_cpu_topology(void) else set_sched_topology(arm64_topology); - reset_cpu_capacity(); - parse_dt_cpu_capacity(); init_sched_energy_costs(); } From dbf8dc017373461bd6475fd32b6fcbfea4d5d2dd Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Fri, 19 Feb 2016 17:06:44 -0800 Subject: [PATCH 74/86] EAS: sched/fair: for SMP bias toward idle core with capacity For SMP devices, on wakeup bias towards idle cores that have capacity vs busy devices that need a higher OPP Signed-off-by: Joe Maples --- kernel/sched/fair.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index e2eb39423ecd7..b0d900a7dd7f2 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5492,8 +5492,8 @@ static inline int find_best_target(struct task_struct *p) /* * Favor 1) busy cpu with most capacity at current OPP - * 2) busy cpu with capacity at higher OPP - * 3) idle_cpu with capacity at current OPP + * 2) idle_cpu with capacity at current OPP + * 3) busy cpu with capacity at higher OPP */ for_each_cpu(i, tsk_cpus_allowed(p)) { @@ -5528,7 +5528,7 @@ static inline int find_best_target(struct task_struct *p) } if (target_cpu < 0) { - target_cpu = backup_cpu >= 0 ? backup_cpu : idle_cpu; + target_cpu = idle_cpu >= 0 ? idle_cpu : backup_cpu; } return target_cpu; From 41cf166556a6b7801743208644d39ebfbff9233c Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Tue, 29 Mar 2016 08:25:58 -0700 Subject: [PATCH 75/86] EAS: sched: fair: fix uninitialized local variable Change-Id: If10e171dc3fd7f2bcc28af2e34d04e6ea08bceb0 Signed-off-by: Joe Maples --- kernel/sched/fair.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index b0d900a7dd7f2..bc7685196b09a 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5486,7 +5486,7 @@ static inline int find_best_target(struct task_struct *p) { int i; int target_cpu = -1; - int target_capacity; + int target_capacity = 0; int idle_cpu = -1; int backup_cpu = -1; From dd371e25ed10b224ddbe4c20301f6a3d5aef7b67 Mon Sep 17 00:00:00 2001 From: Petr Mladek Date: Tue, 11 Oct 2016 13:55:43 -0700 Subject: [PATCH 76/86] kthread: allow to cancel kthread work We are going to use kthread workers more widely and sometimes we will need to make sure that the work is neither pending nor running. This patch implements cancel_*_sync() operations as inspired by workqueues. Well, we are synchronized against the other operations via the worker lock, we use del_timer_sync() and a counter to count parallel cancel operations. Therefore the implementation might be easier. First, we check if a worker is assigned. If not, the work has newer been queued after it was initialized. Second, we take the worker lock. It must be the right one. The work must not be assigned to another worker unless it is initialized in between. Third, we try to cancel the timer when it exists. The timer is deleted synchronously to make sure that the timer call back is not running. We need to temporary release the worker->lock to avoid a possible deadlock with the callback. In the meantime, we set work->canceling counter to avoid any queuing. Fourth, we try to remove the work from a worker list. It might be the list of either normal or delayed works. Fifth, if the work is running, we call kthread_flush_work(). It might take an arbitrary time. We need to release the worker-lock again. In the meantime, we again block any queuing by the canceling counter. As already mentioned, the check for a pending kthread work is done under a lock. In compare with workqueues, we do not need to fight for a single PENDING bit to block other operations. Therefore we do not suffer from the thundering storm problem and all parallel canceling jobs might use kthread_flush_work(). Any queuing is blocked until the counter gets zero. Link: http://lkml.kernel.org/r/1470754545-17632-10-git-send-email-pmladek@suse.com Signed-off-by: Petr Mladek Acked-by: Tejun Heo Cc: Oleg Nesterov Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Steven Rostedt Cc: "Paul E. McKenney" Cc: Josh Triplett Cc: Thomas Gleixner Cc: Jiri Kosina Cc: Borislav Petkov Cc: Michal Hocko Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds [major changes to the original patch while cherry-picking; only rebased the sync variant] Signed-off-by: Juri Lelli Change-Id: I1c9018d76561f1c087f03c53a43670c62b7da45a Signed-off-by: Joe Maples --- include/linux/kthread.h | 4 ++ kernel/kthread.c | 96 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/include/linux/kthread.h b/include/linux/kthread.h index 7dcef3317689e..416d23513b348 100644 --- a/include/linux/kthread.h +++ b/include/linux/kthread.h @@ -75,6 +75,8 @@ struct kthread_work { kthread_work_func_t func; wait_queue_head_t done; struct kthread_worker *worker; + /* Number of canceling calls that are running at the moment. */ + int canceling; }; #define KTHREAD_WORKER_INIT(worker) { \ @@ -137,4 +139,6 @@ bool queue_kthread_work(struct kthread_worker *worker, void flush_kthread_work(struct kthread_work *work); void flush_kthread_worker(struct kthread_worker *worker); +bool kthread_cancel_work_sync(struct kthread_work *work); + #endif /* _LINUX_KTHREAD_H */ diff --git a/kernel/kthread.c b/kernel/kthread.c index c56c6f8ec6078..2ccd5dcfdd310 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -547,6 +547,19 @@ int kthread_worker_fn(void *worker_ptr) } EXPORT_SYMBOL_GPL(kthread_worker_fn); +/* + * Returns true when the work could not be queued at the moment. + * It happens when it is already pending in a worker list + * or when it is being cancelled. + */ +static inline bool queuing_blocked(struct kthread_worker *worker, + struct kthread_work *work) +{ + lockdep_assert_held(&worker->lock); + + return !list_empty(&work->node) || work->canceling; +} + /* insert @work before @pos in @worker */ static void insert_kthread_work(struct kthread_worker *worker, struct kthread_work *work, @@ -576,7 +589,7 @@ bool queue_kthread_work(struct kthread_worker *worker, unsigned long flags; spin_lock_irqsave(&worker->lock, flags); - if (list_empty(&work->node)) { + if (!queuing_blocked(worker, work)) { insert_kthread_work(worker, work, &worker->work_list); ret = true; } @@ -637,6 +650,87 @@ void flush_kthread_work(struct kthread_work *work) } EXPORT_SYMBOL_GPL(flush_kthread_work); +/* + * This function removes the work from the worker queue. Also it makes sure + * that it won't get queued later via the delayed work's timer. + * + * The work might still be in use when this function finishes. See the + * current_work proceed by the worker. + * + * Return: %true if @work was pending and successfully canceled, + * %false if @work was not pending + */ +static bool __kthread_cancel_work(struct kthread_work *work, + unsigned long *flags) +{ + /* + * Try to remove the work from a worker list. It might either + * be from worker->work_list or from worker->delayed_work_list. + */ + if (!list_empty(&work->node)) { + list_del_init(&work->node); + return true; + } + + return false; +} + +static bool __kthread_cancel_work_sync(struct kthread_work *work) +{ + struct kthread_worker *worker = work->worker; + unsigned long flags; + int ret = false; + + if (!worker) + goto out; + + spin_lock_irqsave(&worker->lock, flags); + /* Work must not be used with >1 worker, see kthread_queue_work(). */ + WARN_ON_ONCE(work->worker != worker); + + ret = __kthread_cancel_work(work, &flags); + + if (worker->current_work != work) + goto out_fast; + + /* + * The work is in progress and we need to wait with the lock released. + * In the meantime, block any queuing by setting the canceling counter. + */ + work->canceling++; + spin_unlock_irqrestore(&worker->lock, flags); + flush_kthread_work(work); + spin_lock_irqsave(&worker->lock, flags); + work->canceling--; + +out_fast: + spin_unlock_irqrestore(&worker->lock, flags); +out: + return ret; +} + +/** + * kthread_cancel_work_sync - cancel a kthread work and wait for it to finish + * @work: the kthread work to cancel + * + * Cancel @work and wait for its execution to finish. This function + * can be used even if the work re-queues itself. On return from this + * function, @work is guaranteed to be not pending or executing on any CPU. + * + * kthread_cancel_work_sync(&delayed_work->work) must not be used for + * delayed_work's. Use kthread_cancel_delayed_work_sync() instead. + * + * The caller must ensure that the worker on which @work was last + * queued can't be destroyed before this function returns. + * + * Return: %true if @work was pending, %false otherwise. + */ +bool kthread_cancel_work_sync(struct kthread_work *work) +{ + return __kthread_cancel_work_sync(work); +} +EXPORT_SYMBOL_GPL(kthread_cancel_work_sync); + /** * flush_kthread_worker - flush all current works on a kthread_worker * @worker: worker to flush From fed1f07b91cc684ef3c39836807054f8af543489 Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Thu, 25 Feb 2016 13:06:52 -0800 Subject: [PATCH 77/86] sched/cpufreq_sched: use shorter throttle for raising OPP Avoid cases where a brief drop in load causes a change to a low OPP for the full throttle period. Use a shorter throttle period for raising OPP than for lowering OPP. Signed-off-by: Joe Maples --- kernel/sched/cpufreq_sched.c | 42 ++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/kernel/sched/cpufreq_sched.c b/kernel/sched/cpufreq_sched.c index cd965f066cce6..59ec230a41d9d 100644 --- a/kernel/sched/cpufreq_sched.c +++ b/kernel/sched/cpufreq_sched.c @@ -14,7 +14,8 @@ #include "sched.h" -#define THROTTLE_NSEC 20000000 /* 50ms default */ +#define THROTTLE_DOWN_NSEC 20000000 /* 20ms default */ +#define THROTTLE_UP_NSEC 500000 /* 500us default */ static DEFINE_PER_CPU(unsigned long, pcpu_capacity); static DEFINE_PER_CPU(struct cpufreq_policy *, pcpu_policy); @@ -22,9 +23,13 @@ static DEFINE_PER_CPU(int, governor_started); /** * gov_data - per-policy data internal to the governor - * @throttle: next throttling period expiry. Derived from throttle_nsec - * @throttle_nsec: throttle period length in nanoseconds - * @freq: new frequency stored in *_sched_update_cpu and used in *_sched_thread + * @up_throttle: next throttling period expiry if increasing OPP + * @down_throttle: next throttling period expiry if decreasing OPP + * @up_throttle_nsec: throttle period length in nanoseconds if increasing OPP + * @down_throttle_nsec: throttle period length in nanoseconds if decreasing OPP + * @task: worker thread for dvfs transition that may block/sleep + * @irq_work: callback used to wake up worker thread + * @requested_freq: last frequency requested by the sched governor * * struct gov_data is the per-policy cpufreq_sched-specific data structure. A * per-policy instance of it is created when the cpufreq_sched governor receives @@ -35,8 +40,10 @@ static DEFINE_PER_CPU(int, governor_started); * call down_write(policy->rwsem). */ struct gov_data { - ktime_t throttle; - unsigned int throttle_nsec; + ktime_t up_throttle; + ktime_t down_throttle; + unsigned int up_throttle_nsec; + unsigned int down_throttle_nsec; struct cpufreq_policy *policy; unsigned int freq; bool change_pending; @@ -67,7 +74,8 @@ static void cpufreq_sched_try_driver_target(struct cpufreq_policy *policy, unsig __cpufreq_driver_target(policy, freq, CPUFREQ_RELATION_L); - gd->throttle = ktime_add_ns(ktime_get(), gd->throttle_nsec); + gd->up_throttle = ktime_add_ns(ktime_get(), gd->up_throttle_nsec); + gd->down_throttle = ktime_add_ns(ktime_get(), gd->down_throttle_nsec); up_write(&policy->rwsem); } @@ -171,8 +179,11 @@ void cpufreq_sched_set_cap(int cpu, unsigned long capacity) gd = policy->governor_data; + ktime_t throttle = gd->requested_freq < cur_freq ? + gd->down_throttle : gd->up_throttle; + /* bail early if we are throttled */ - if (ktime_compare(ktime_get(), gd->throttle) < 0) + if (ktime_compare(ktime_get(), throttle) < 0) goto out; /* find max capacity requested by cpus in this policy */ @@ -300,19 +311,12 @@ static int cpufreq_sched_policy_init(struct cpufreq_policy *policy) goto err; } - /* - * Don't ask for freq changes at an higher rate than what - * the driver advertises as transition latency. - */ -#ifdef THROTTLE_FROM_CPUFREQ_DRIVER_LATENCY - gd->throttle_nsec = policy->cpuinfo.transition_latency ? + gd->up_throttle_nsec = policy->cpuinfo.transition_latency ? policy->cpuinfo.transition_latency : - THROTTLE_NSEC; -#else - gd->throttle_nsec = THROTTLE_NSEC; -#endif + THROTTLE_UP_NSEC; + gd->down_throttle_nsec = THROTTLE_DOWN_NSEC; pr_debug("%s: throttle threshold = %u [ns]\n", - __func__, gd->throttle_nsec); + __func__, gd->up_throttle_nsec); policy->governor_data = gd; gd->policy = policy; From c43e344557e98f3600ab3800c653aa0bfd78dfaa Mon Sep 17 00:00:00 2001 From: Srinath Sridharan Date: Wed, 4 May 2016 12:45:22 -0700 Subject: [PATCH 78/86] sched/cpufreq: Increasing throttle_down_nsec to 50ms Change-Id: I2d8969cf2a64fa719b9dd86f43f9dd14b1ff84fe Signed-off-by: Joe Maples --- kernel/sched/cpufreq_sched.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/cpufreq_sched.c b/kernel/sched/cpufreq_sched.c index 59ec230a41d9d..7bbc47131d260 100644 --- a/kernel/sched/cpufreq_sched.c +++ b/kernel/sched/cpufreq_sched.c @@ -14,7 +14,7 @@ #include "sched.h" -#define THROTTLE_DOWN_NSEC 20000000 /* 20ms default */ +#define THROTTLE_DOWN_NSEC 50000000 /* 50ms default */ #define THROTTLE_UP_NSEC 500000 /* 500us default */ static DEFINE_PER_CPU(unsigned long, pcpu_capacity); From e73781a09e04c7cb8068b01b42aebdd76c1edf3c Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Wed, 11 May 2016 11:59:00 -0700 Subject: [PATCH 79/86] sched-freq: make throttle times tunable Change-Id: I127879645367425b273441d7f0306bb15d5633cb Signed-off-by: Joe Maples --- kernel/sched/cpufreq_sched.c | 37 +++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/kernel/sched/cpufreq_sched.c b/kernel/sched/cpufreq_sched.c index 7bbc47131d260..1065ee292b743 100644 --- a/kernel/sched/cpufreq_sched.c +++ b/kernel/sched/cpufreq_sched.c @@ -179,7 +179,7 @@ void cpufreq_sched_set_cap(int cpu, unsigned long capacity) gd = policy->governor_data; - ktime_t throttle = gd->requested_freq < cur_freq ? + ktime_t throttle = gd->freq < policy->cur ? gd->down_throttle : gd->up_throttle; /* bail early if we are throttled */ @@ -350,6 +350,8 @@ static int cpufreq_sched_policy_exit(struct cpufreq_policy *policy) clear_sched_energy_freq(); + sysfs_remove_group(get_governor_parent_kobj(policy), get_sysfs_attr()); + policy->governor_data = NULL; mutex_lock(&gov_list_lock); list_del(&gd->gov_list); @@ -398,12 +400,12 @@ static int cpufreq_sched_setup(struct cpufreq_policy *policy, unsigned int event } /* Tunables */ -static ssize_t show_throttle_ns(struct gov_data *gd, char *buf) +static ssize_t show_up_throttle_nsec(struct gov_data *gd, char *buf) { - return sprintf(buf, "%u\n", gd->throttle_nsec); + return sprintf(buf, "%u\n", gd->up_throttle_nsec); } -static ssize_t store_throttle_ns(struct gov_data *gd, +static ssize_t store_up_throttle_nsec(struct gov_data *gd, const char *buf, size_t count) { int ret; @@ -412,9 +414,28 @@ static ssize_t store_throttle_ns(struct gov_data *gd, ret = kstrtoul(buf, 0, &val); if (ret < 0) return ret; - gd->throttle_nsec = val; + gd->up_throttle_nsec = val; return count; } + +static ssize_t show_down_throttle_nsec(struct gov_data *gd, char *buf) +{ + return sprintf(buf, "%u\n", gd->down_throttle_nsec); +} + +static ssize_t store_down_throttle_nsec(struct gov_data *gd, + const char *buf, size_t count) +{ + int ret; + long unsigned int val; + + ret = kstrtoul(buf, 0, &val); + if (ret < 0) + return ret; + gd->down_throttle_nsec = val; + return count; +} + /* * Create show/store routines * - sys: One governor instance for complete SYSTEM @@ -446,11 +467,13 @@ static ssize_t store_##file_name##_gov_pol \ store_gov_pol_sys(file_name); \ gov_pol_attr_rw(file_name) -tunable_handlers(throttle_ns); +tunable_handlers(down_throttle_nsec); +tunable_handlers(up_throttle_nsec); /* Per policy governor instance */ static struct attribute *sched_attributes_gov_pol[] = { - &throttle_ns_gov_pol.attr, + &up_throttle_nsec_gov_pol.attr, + &down_throttle_nsec_gov_pol.attr, NULL, }; From 48a70666caf358d1803aa31c5aba0ac734f3a7e0 Mon Sep 17 00:00:00 2001 From: mydongistiny Date: Mon, 12 Jun 2017 02:01:01 -0700 Subject: [PATCH 80/86] arm: dts: remove unneeded bootargs for eas Signed-off-by: mydongistiny Signed-off-by: Joe Maples --- arch/arm/boot/dts/qcom/msm8909.dtsi | 2 +- arch/arm/boot/dts/qcom/msm8916.dtsi | 2 +- arch/arm/boot/dts/qcom/msm8936.dtsi | 2 +- arch/arm/boot/dts/qcom/msm8939.dtsi | 2 +- arch/arm/boot/dts/qcom/msm8992.dtsi | 2 +- arch/arm/boot/dts/qcom/msm8994-v2.0.dtsi | 2 +- arch/arm/boot/dts/qcom/msm8994-v2.1.dtsi | 2 +- arch/arm/boot/dts/qcom/msm8994.dtsi | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/arm/boot/dts/qcom/msm8909.dtsi b/arch/arm/boot/dts/qcom/msm8909.dtsi index 5ff5818507db5..deb8b7c185120 100644 --- a/arch/arm/boot/dts/qcom/msm8909.dtsi +++ b/arch/arm/boot/dts/qcom/msm8909.dtsi @@ -24,7 +24,7 @@ interrupt-parent = <&intc>; chosen { - bootargs = "sched_enable_hmp=1"; + bootargs = ""; }; aliases { diff --git a/arch/arm/boot/dts/qcom/msm8916.dtsi b/arch/arm/boot/dts/qcom/msm8916.dtsi index 05e67fb7ca280..2284449b8d571 100644 --- a/arch/arm/boot/dts/qcom/msm8916.dtsi +++ b/arch/arm/boot/dts/qcom/msm8916.dtsi @@ -25,7 +25,7 @@ interrupt-parent = <&intc>; chosen { - bootargs = "sched_enable_hmp=1"; + bootargs = ""; }; aliases { diff --git a/arch/arm/boot/dts/qcom/msm8936.dtsi b/arch/arm/boot/dts/qcom/msm8936.dtsi index b741a0471a9a8..e10ebfc52b212 100644 --- a/arch/arm/boot/dts/qcom/msm8936.dtsi +++ b/arch/arm/boot/dts/qcom/msm8936.dtsi @@ -26,7 +26,7 @@ qcom,msm-id = <233 0>, <240 0>, <242 0>; chosen { - bootargs = "boot_cpus=0,1 sched_enable_hmp=1"; + bootargs = "boot_cpus=0,1"; }; }; diff --git a/arch/arm/boot/dts/qcom/msm8939.dtsi b/arch/arm/boot/dts/qcom/msm8939.dtsi index ae94919dbd8a7..c9144730cff85 100644 --- a/arch/arm/boot/dts/qcom/msm8939.dtsi +++ b/arch/arm/boot/dts/qcom/msm8939.dtsi @@ -26,7 +26,7 @@ qcom,msm-id = <239 0>, <241 0>, <263 0>; chosen { - bootargs = "boot_cpus=0,4,5,6,7 sched_enable_hmp=1"; + bootargs = "boot_cpus=0,4,5,6,7"; }; }; diff --git a/arch/arm/boot/dts/qcom/msm8992.dtsi b/arch/arm/boot/dts/qcom/msm8992.dtsi index 3021261625958..475a29a2e0bf7 100644 --- a/arch/arm/boot/dts/qcom/msm8992.dtsi +++ b/arch/arm/boot/dts/qcom/msm8992.dtsi @@ -23,7 +23,7 @@ interrupt-parent = <&intc>; chosen { - bootargs = "sched_enable_hmp=1 sched_enable_power_aware=1"; + bootargs = ""; }; aliases { diff --git a/arch/arm/boot/dts/qcom/msm8994-v2.0.dtsi b/arch/arm/boot/dts/qcom/msm8994-v2.0.dtsi index b824d0c5d195c..da8ff11b0c37f 100644 --- a/arch/arm/boot/dts/qcom/msm8994-v2.0.dtsi +++ b/arch/arm/boot/dts/qcom/msm8994-v2.0.dtsi @@ -24,7 +24,7 @@ qcom,msm-id = <207 0x20000>; chosen { - bootargs = "sched_enable_hmp=1 sched_enable_power_aware=1"; + bootargs = ""; }; }; diff --git a/arch/arm/boot/dts/qcom/msm8994-v2.1.dtsi b/arch/arm/boot/dts/qcom/msm8994-v2.1.dtsi index 93bc398c9693f..4959cfa396513 100644 --- a/arch/arm/boot/dts/qcom/msm8994-v2.1.dtsi +++ b/arch/arm/boot/dts/qcom/msm8994-v2.1.dtsi @@ -24,7 +24,7 @@ qcom,msm-id = <207 0x20001>; chosen { - bootargs = "sched_enable_hmp=1 sched_enable_power_aware=1"; + bootargs = ""; }; }; diff --git a/arch/arm/boot/dts/qcom/msm8994.dtsi b/arch/arm/boot/dts/qcom/msm8994.dtsi index 215943fa20abd..15d1271ef3076 100644 --- a/arch/arm/boot/dts/qcom/msm8994.dtsi +++ b/arch/arm/boot/dts/qcom/msm8994.dtsi @@ -24,7 +24,7 @@ interrupt-parent = <&intc>; chosen { - bootargs = "sched_enable_hmp=1 sched_enable_power_aware=1"; + bootargs = ""; }; aliases { From a1987e58842bfbebbda284a4c33352a2c1bdf477 Mon Sep 17 00:00:00 2001 From: Tristan Marsell Date: Sat, 10 Jun 2017 14:55:56 +0200 Subject: [PATCH 81/86] dtsi: Add EAS Energy Profile --- arch/arm/boot/dts/qcom/msm8994.dtsi | 73 +++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/arch/arm/boot/dts/qcom/msm8994.dtsi b/arch/arm/boot/dts/qcom/msm8994.dtsi index 15d1271ef3076..679571513d8df 100644 --- a/arch/arm/boot/dts/qcom/msm8994.dtsi +++ b/arch/arm/boot/dts/qcom/msm8994.dtsi @@ -87,6 +87,7 @@ qcom,acc = <&acc0>; qcom,ldo = <&ldo0>; next-level-cache = <&L2_0>; + sched-energy-costs = <&CPU_COST_0 &CLUSTER_COST_0>; L2_0: l2-cache { compatible = "arm,arch-cache"; cache-level = <2>; @@ -114,6 +115,7 @@ qcom,acc = <&acc1>; qcom,ldo = <&ldo1>; next-level-cache = <&L2_0>; + sched-energy-costs = <&CPU_COST_0 &CLUSTER_COST_0>; L1_I_1: l1-icache { compatible = "arm,arch-cache"; qcom,dump-size = <0x9040>; @@ -132,6 +134,7 @@ qcom,acc = <&acc2>; qcom,ldo = <&ldo2>; next-level-cache = <&L2_0>; + sched-energy-costs = <&CPU_COST_0 &CLUSTER_COST_0>; L1_I_2: l1-icache { compatible = "arm,arch-cache"; qcom,dump-size = <0x9040>; @@ -150,6 +153,7 @@ qcom,acc = <&acc3>; qcom,ldo = <&ldo3>; next-level-cache = <&L2_0>; + sched-energy-costs = <&CPU_COST_0 &CLUSTER_COST_0>; L1_I_3: l1-icache { compatible = "arm,arch-cache"; qcom,dump-size = <0x9040>; @@ -168,6 +172,7 @@ qcom,acc = <&acc4>; qcom,ldo = <&ldo4>; next-level-cache = <&L2_1>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; L2_1: l2-cache { compatible = "arm,arch-cache"; cache-level = <2>; @@ -201,6 +206,7 @@ qcom,acc = <&acc5>; qcom,ldo = <&ldo5>; next-level-cache = <&L2_1>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; L1_itlb_101: l1-itlb { qcom,dump-size = <0x400>; }; @@ -225,6 +231,7 @@ qcom,acc = <&acc6>; qcom,ldo = <&ldo6>; next-level-cache = <&L2_1>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; L1_itlb_102: l1-itlb { qcom,dump-size = <0x400>; }; @@ -249,6 +256,7 @@ qcom,acc = <&acc7>; qcom,ldo = <&ldo7>; next-level-cache = <&L2_1>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; L1_itlb_103: l1-itlb { qcom,dump-size = <0x400>; }; @@ -264,6 +272,71 @@ qcom,dump-size = <0x9040>; }; }; + + energy-costs { + CPU_COST_0: core-cost0 { + busy-cost-data = < + 184 48 + 266 66 + 378 115 + 419 137 + 450 156 + 532 217 + 573 251 + 593 273 + >; + idle-cost-data = < + 0 + >; + }; + CPU_COST_1: core-cost1 { + busy-cost-data = < + 256 155 + 358 174 + 512 289 + 614 428 + 716 602 + 829 835 + 921 1098 + 1024 1382 + >; + idle-cost-data = < + 0 + >; + }; + CLUSTER_COST_0: cluster-cost0 { + busy-cost-data = < + 184 74 + 266 104 + 378 151 + 419 168 + 450 181 + 532 220 + 573 238 + 593 250 + >; + idle-cost-data = < + 110 + 105 + >; + }; + CLUSTER_COST_1: cluster-cost1 { + busy-cost-data = < + 256 90 + 358 127 + 512 174 + 614 201 + 716 222 + 829 240 + 921 252 + 1024 259 + >; + idle-cost-data = < + 267 + 231 + >; + }; + }; }; soc: soc { }; From 102b8bd7824339d92fb721a12886e4da208f7277 Mon Sep 17 00:00:00 2001 From: Tristan Marsell Date: Sun, 11 Jun 2017 11:22:45 +0200 Subject: [PATCH 82/86] Update EAS Energy Profile to Angler --- arch/arm/boot/dts/qcom/msm8994.dtsi | 88 ++++++++++++++++------------- 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/arch/arm/boot/dts/qcom/msm8994.dtsi b/arch/arm/boot/dts/qcom/msm8994.dtsi index 679571513d8df..ae079b3f121ec 100644 --- a/arch/arm/boot/dts/qcom/msm8994.dtsi +++ b/arch/arm/boot/dts/qcom/msm8994.dtsi @@ -275,65 +275,75 @@ energy-costs { CPU_COST_0: core-cost0 { - busy-cost-data = < - 184 48 - 266 66 - 378 115 - 419 137 - 450 156 - 532 217 - 573 251 - 593 273 + busy-cost-data = < + 136 7 + 174 16 + 212 26 + 245 39 + 278 53 + 309 76 + 339 102 + 424 161 + 509 227 >; idle-cost-data = < - 0 + 33 25 23 18 >; }; CPU_COST_1: core-cost1 { busy-cost-data = < - 256 155 - 358 174 - 512 289 - 614 428 - 716 602 - 829 835 - 921 1098 - 1024 1382 + 220 306 + 290 320 + 360 334 + 425 432 + 490 533 + 599 721 + 708 917 + 762 1058 + 817 1203 + 871 1410 + 926 1612 + 975 1868 + 1024 2126 >; idle-cost-data = < - 0 + 65 60 28 21 >; }; CLUSTER_COST_0: cluster-cost0 { busy-cost-data = < - 184 74 - 266 104 - 378 151 - 419 168 - 450 181 - 532 220 - 573 238 - 593 250 + 136 14 + 174 27 + 212 39 + 245 52 + 278 62 + 309 79 + 339 94 + 424 132 + 509 165 >; idle-cost-data = < - 110 - 105 + 0 0 0 0 >; }; CLUSTER_COST_1: cluster-cost1 { busy-cost-data = < - 256 90 - 358 127 - 512 174 - 614 201 - 716 222 - 829 240 - 921 252 - 1024 259 + 220 80 + 290 80 + 360 80 + 425 80 + 490 80 + 599 80 + 708 80 + 762 80 + 817 80 + 871 80 + 926 80 + 975 80 + 1024 80 >; idle-cost-data = < - 267 - 231 + 0 0 0 0 >; }; }; From f4ea1cec745c5b1df46bb472ec8c56fe63fc95e7 Mon Sep 17 00:00:00 2001 From: "www.anuprita804@gmail.com" Date: Fri, 18 Aug 2017 19:48:36 +0200 Subject: [PATCH 83/86] sched: fix build error --- include/linux/sched.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/linux/sched.h b/include/linux/sched.h index ef6fe7558cd8b..b389a1292adf9 100755 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1341,6 +1341,11 @@ struct task_struct { atomic_t usage; unsigned int flags; /* per process flags, defined below */ unsigned int ptrace; + #ifdef VENDOR_EDIT + //huruihuan add for kill task in D status + unsigned int kill_flag; + struct timespec ttu; + #endif #ifdef CONFIG_SMP struct llist_node wake_entry; From 4abfa02d0cf336da1b6967ad043e4629ab2024d6 Mon Sep 17 00:00:00 2001 From: "www.anuprita804@gmail.com" Date: Fri, 18 Aug 2017 20:06:11 +0200 Subject: [PATCH 84/86] fix unterminated endif --- kernel/sched/debug.c | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 14c0967e7218b..bde6dd63d6698 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -210,7 +210,6 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) cfs_rq->nr_spread_over); SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running); SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight); -#ifdef CONFIG_FAIR_GROUP_SCHED #ifdef CONFIG_SMP SEQ_printf(m, " .%-30s: %ld\n", "runnable_load_avg", cfs_rq->runnable_load_avg); From 8657a5d512a1e295e56fb4ee3b541575fe546533 Mon Sep 17 00:00:00 2001 From: jollaman999 Date: Sun, 7 Aug 2016 10:23:49 +0200 Subject: [PATCH 85/86] Get rid of __cpuinit - squashed (jollaman999) Remove one of miss removed __cpuinitdata kbuild: drop cpuinit/cpuexit checks in modpost With the removal of cpuinit/cpuexit we no longer need to do section mismatch checks on these sections. Drop it from modpost. Signed-off-by: Dennis Rassmann cpuinit: turn __cpuinit into nops now that there's no users left, turn __cpuinit into a nop for now, to avoid breaking any in-flight code. In a few releases we can just drop it entirely. Signed-off-by: Arjan van de Ven Signed-off-by: Dennis Rassmann Get rid of __cpuinit This commit is the result of find . -name '*.c' | xargs sed -i 's/ __cpuinit / /g' find . -name '*.c' | xargs sed -i 's/ __cpuexit / /g' find . -name '*.c' | xargs sed -i 's/ __cpuinitdata / /g' find . -name '*.c' | xargs sed -i 's/ __cpuinit$//g' find ./arch/ -name '*.h' | xargs sed -i 's/ __cpuinit//g' find . -name '*.c' | xargs sed -i 's/^__cpuinit //g' find . -name '*.c' | xargs sed -i 's/^__cpuinitdata //g' find . -name '*.c' | xargs sed -i 's/\*__cpuinit /\*/g' find . -name '*.c' | xargs sed -i 's/ __cpuinitconst / /g' find . -name '*.h' | xargs sed -i 's/ __cpuinit / /g' find . -name '*.h' | xargs sed -i 's/ __cpuinitdata / /g' git add . git reset include/linux/init.h git checkout -- include/linux/init.h Change-Id: Idf1210c65a0a3936b96d10c8aeae63f1f6809c45 Signed-off-by: mydongistiny --- arch/alpha/kernel/smp.c | 10 +-- arch/alpha/kernel/traps.c | 4 +- arch/arc/include/asm/irq.h | 2 +- arch/arc/kernel/irq.c | 2 +- arch/arc/kernel/setup.c | 8 +- arch/arc/kernel/smp.c | 4 +- arch/arc/kernel/time.c | 6 +- arch/arc/mm/cache_arc700.c | 4 +- arch/arc/mm/tlb.c | 4 +- arch/arm/common/mcpm_platsmp.c | 4 +- arch/arm/kernel/hw_breakpoint.c | 4 +- arch/arm/kernel/perf_event_cpu.c | 4 +- arch/arm/kernel/psci_smp.c | 2 +- arch/arm/kernel/smp.c | 18 ++-- arch/arm/kernel/smp_twd.c | 6 +- arch/arm/lib/delay.c | 2 +- arch/arm/mach-exynos/platsmp.c | 4 +- arch/arm/mach-highbank/platsmp.c | 2 +- arch/arm/mach-imx/platsmp.c | 2 +- arch/arm/mach-msm/platsmp.c | 28 +++--- arch/arm/mach-msm/timer.c | 2 +- arch/arm/mach-mvebu/platsmp.c | 4 +- arch/arm/mach-omap2/omap-mpuss-lowpower.c | 2 +- arch/arm/mach-omap2/omap-smp.c | 4 +- arch/arm/mach-omap2/omap-wakeupgen.c | 2 +- arch/arm/mach-prima2/platsmp.c | 4 +- arch/arm/mach-shmobile/smp-emev2.c | 2 +- arch/arm/mach-shmobile/smp-r8a7779.c | 2 +- arch/arm/mach-shmobile/smp-sh73a0.c | 2 +- arch/arm/mach-socfpga/platsmp.c | 2 +- arch/arm/mach-spear/generic.h | 2 +- arch/arm/mach-spear/platsmp.c | 4 +- arch/arm/mach-tegra/platsmp.c | 4 +- arch/arm/mach-tegra/pm.c | 2 +- arch/arm/mach-ux500/platsmp.c | 4 +- arch/arm/mach-zynq/common.h | 2 +- arch/arm/mach-zynq/platsmp.c | 6 +- arch/arm/plat-versatile/platsmp.c | 6 +- arch/arm64/kernel/debug-monitors.c | 6 +- arch/arm64/kernel/hw_breakpoint.c | 4 +- arch/arm64/kernel/perf_event.c | 4 +- arch/arm64/kernel/smp.c | 6 +- arch/blackfin/kernel/perf_event.c | 2 +- arch/blackfin/kernel/setup.c | 4 +- arch/blackfin/mach-bf561/smp.c | 6 +- arch/blackfin/mach-common/cache-c.c | 4 +- arch/blackfin/mach-common/ints-priority.c | 2 +- arch/blackfin/mach-common/smp.c | 12 +-- arch/cris/arch-v32/kernel/smp.c | 2 +- arch/frv/kernel/setup.c | 2 +- arch/hexagon/kernel/setup.c | 2 +- arch/hexagon/kernel/smp.c | 4 +- arch/ia64/kernel/acpi.c | 4 +- arch/ia64/kernel/err_inject.c | 8 +- arch/ia64/kernel/mca.c | 12 +-- arch/ia64/kernel/numa.c | 4 +- arch/ia64/kernel/palinfo.c | 4 +- arch/ia64/kernel/salinfo.c | 4 +- arch/ia64/kernel/setup.c | 10 +-- arch/ia64/kernel/smpboot.c | 8 +- arch/ia64/kernel/topology.c | 16 ++-- arch/ia64/mm/contig.c | 2 +- arch/ia64/mm/discontig.c | 2 +- arch/ia64/mm/numa.c | 2 +- arch/ia64/sn/kernel/setup.c | 8 +- arch/ia64/xen/hypervisor.c | 2 +- arch/m32r/kernel/smpboot.c | 2 +- arch/metag/kernel/perf/perf_event.c | 4 +- arch/metag/kernel/smp.c | 16 ++-- arch/metag/kernel/traps.c | 2 +- arch/mips/ath79/setup.c | 2 +- arch/mips/cavium-octeon/octeon-irq.c | 12 +-- arch/mips/cavium-octeon/smp.c | 6 +- arch/mips/include/asm/uasm.h | 4 +- arch/mips/kernel/cevt-bcm1480.c | 2 +- arch/mips/kernel/cevt-gic.c | 2 +- arch/mips/kernel/cevt-r4k.c | 2 +- arch/mips/kernel/cevt-sb1250.c | 2 +- arch/mips/kernel/cevt-smtc.c | 2 +- arch/mips/kernel/cpu-probe.c | 14 +-- arch/mips/kernel/smp-bmips.c | 6 +- arch/mips/kernel/smp-mt.c | 6 +- arch/mips/kernel/smp-up.c | 6 +- arch/mips/kernel/smp.c | 6 +- arch/mips/kernel/smtc.c | 2 +- arch/mips/kernel/spram.c | 14 +-- arch/mips/kernel/sync-r4k.c | 12 +-- arch/mips/kernel/traps.c | 10 +-- arch/mips/kernel/watch.c | 2 +- arch/mips/lantiq/irq.c | 2 +- arch/mips/lib/uncached.c | 2 +- arch/mips/mm/c-octeon.c | 6 +- arch/mips/mm/c-r3k.c | 8 +- arch/mips/mm/c-r4k.c | 34 +++---- arch/mips/mm/c-tx39.c | 2 +- arch/mips/mm/cache.c | 2 +- arch/mips/mm/page.c | 20 ++--- arch/mips/mm/sc-ip22.c | 2 +- arch/mips/mm/sc-mips.c | 2 +- arch/mips/mm/sc-r5k.c | 2 +- arch/mips/mm/sc-rm7k.c | 12 +-- arch/mips/mm/tlb-r3k.c | 2 +- arch/mips/mm/tlb-r4k.c | 4 +- arch/mips/mm/tlb-r8k.c | 4 +- arch/mips/mm/tlbex.c | 90 +++++++++---------- arch/mips/mti-malta/malta-smtc.c | 6 +- arch/mips/mti-malta/malta-time.c | 2 +- arch/mips/mti-sead3/sead3-time.c | 2 +- arch/mips/netlogic/common/smp.c | 4 +- arch/mips/netlogic/common/time.c | 2 +- arch/mips/netlogic/xlr/wakeup.c | 2 +- arch/mips/pci/pci-ip27.c | 2 +- arch/mips/pmcs-msp71xx/msp_smtc.c | 6 +- arch/mips/pmcs-msp71xx/msp_time.c | 2 +- arch/mips/pnx833x/common/interrupts.c | 2 +- arch/mips/powertv/time.c | 2 +- arch/mips/ralink/irq.c | 2 +- arch/mips/sgi-ip27/ip27-init.c | 4 +- arch/mips/sgi-ip27/ip27-smp.c | 6 +- arch/mips/sgi-ip27/ip27-timer.c | 6 +- arch/mips/sgi-ip27/ip27-xtalk.c | 6 +- arch/mips/sibyte/bcm1480/smp.c | 8 +- arch/mips/sibyte/sb1250/smp.c | 8 +- arch/openrisc/kernel/setup.c | 2 +- arch/parisc/kernel/firmware.c | 12 +-- arch/parisc/kernel/hardware.c | 2 +- arch/parisc/kernel/processor.c | 6 +- arch/parisc/kernel/smp.c | 6 +- arch/powerpc/include/asm/rtas.h | 4 +- arch/powerpc/include/asm/vdso.h | 2 +- arch/powerpc/kernel/cacheinfo.c | 30 +++---- arch/powerpc/kernel/rtas.c | 4 +- arch/powerpc/kernel/smp.c | 4 +- arch/powerpc/kernel/sysfs.c | 6 +- arch/powerpc/kernel/time.c | 2 +- arch/powerpc/kernel/vdso.c | 2 +- arch/powerpc/mm/44x_mmu.c | 6 +- arch/powerpc/mm/hash_utils_64.c | 2 +- arch/powerpc/mm/mmu_context_nohash.c | 4 +- arch/powerpc/mm/numa.c | 6 +- arch/powerpc/mm/tlb_nohash.c | 2 +- arch/powerpc/perf/core-book3s.c | 4 +- arch/powerpc/platforms/44x/currituck.c | 4 +- arch/powerpc/platforms/44x/iss4xx.c | 4 +- arch/powerpc/platforms/85xx/smp.c | 6 +- arch/powerpc/platforms/powermac/smp.c | 2 +- arch/powerpc/platforms/powernv/smp.c | 2 +- arch/s390/kernel/cache.c | 10 +-- arch/s390/kernel/perf_cpum_cf.c | 2 +- arch/s390/kernel/processor.c | 2 +- arch/s390/kernel/smp.c | 14 +-- arch/s390/kernel/sysinfo.c | 2 +- arch/s390/kernel/vtime.c | 4 +- arch/s390/mm/fault.c | 2 +- arch/score/mm/tlb-score.c | 2 +- arch/sh/kernel/cpu/init.c | 18 ++-- arch/sh/kernel/cpu/sh2/probe.c | 2 +- arch/sh/kernel/cpu/sh2a/probe.c | 2 +- arch/sh/kernel/cpu/sh3/probe.c | 2 +- arch/sh/kernel/cpu/sh4/probe.c | 2 +- arch/sh/kernel/cpu/sh4a/smp-shx3.c | 6 +- arch/sh/kernel/cpu/sh5/probe.c | 2 +- arch/sh/kernel/perf_event.c | 4 +- arch/sh/kernel/process.c | 2 +- arch/sh/kernel/setup.c | 2 +- arch/sh/kernel/smp.c | 8 +- arch/sh/kernel/traps_32.c | 2 +- arch/sh/kernel/traps_64.c | 2 +- arch/sh/mm/tlb-sh5.c | 2 +- arch/sparc/kernel/ds.c | 4 +- arch/sparc/kernel/entry.h | 2 +- arch/sparc/kernel/irq_64.c | 4 +- arch/sparc/kernel/leon_smp.c | 10 +-- arch/sparc/kernel/mdesc.c | 26 +++--- arch/sparc/kernel/smp_32.c | 20 ++--- arch/sparc/kernel/smp_64.c | 8 +- arch/sparc/kernel/sun4d_smp.c | 6 +- arch/sparc/kernel/sun4m_smp.c | 6 +- arch/sparc/kernel/sysfs.c | 4 +- arch/sparc/mm/init_64.c | 2 +- arch/sparc/mm/srmmu.c | 12 +-- arch/tile/kernel/irq.c | 2 +- arch/tile/kernel/messaging.c | 2 +- arch/tile/kernel/setup.c | 12 +-- arch/tile/kernel/smpboot.c | 6 +- arch/tile/kernel/time.c | 2 +- arch/x86/include/asm/cpu.h | 2 +- arch/x86/include/asm/microcode.h | 4 +- arch/x86/include/asm/microcode_intel.h | 4 +- arch/x86/include/asm/mmconfig.h | 4 +- arch/x86/include/asm/mpspec.h | 2 +- arch/x86/include/asm/numa.h | 6 +- arch/x86/include/asm/prom.h | 2 +- arch/x86/include/asm/smp.h | 2 +- arch/x86/kernel/acpi/boot.c | 6 +- arch/x86/kernel/apic/apic.c | 28 +++--- arch/x86/kernel/apic/apic_numachip.c | 2 +- arch/x86/kernel/apic/es7000_32.c | 2 +- arch/x86/kernel/apic/numaq_32.c | 2 +- arch/x86/kernel/apic/x2apic_cluster.c | 2 +- arch/x86/kernel/apic/x2apic_uv_x.c | 12 +-- arch/x86/kernel/cpu/amd.c | 32 +++---- arch/x86/kernel/cpu/centaur.c | 26 +++--- arch/x86/kernel/cpu/common.c | 56 ++++++------ arch/x86/kernel/cpu/cyrix.c | 40 ++++----- arch/x86/kernel/cpu/hypervisor.c | 2 +- arch/x86/kernel/cpu/intel.c | 30 +++---- arch/x86/kernel/cpu/intel_cacheinfo.c | 52 +++++------ arch/x86/kernel/cpu/mcheck/mce.c | 22 ++--- arch/x86/kernel/cpu/mcheck/mce_amd.c | 10 +-- arch/x86/kernel/cpu/mcheck/therm_throt.c | 8 +- arch/x86/kernel/cpu/perf_event.c | 2 +- arch/x86/kernel/cpu/perf_event_amd_ibs.c | 2 +- arch/x86/kernel/cpu/perf_event_amd_uncore.c | 24 ++--- arch/x86/kernel/cpu/perf_event_intel_uncore.c | 18 ++-- arch/x86/kernel/cpu/rdrand.c | 2 +- arch/x86/kernel/cpu/scattered.c | 4 +- arch/x86/kernel/cpu/topology.c | 2 +- arch/x86/kernel/cpu/transmeta.c | 6 +- arch/x86/kernel/cpu/umc.c | 2 +- arch/x86/kernel/cpu/vmware.c | 2 +- arch/x86/kernel/cpuid.c | 4 +- arch/x86/kernel/devicetree.c | 2 +- arch/x86/kernel/i387.c | 6 +- arch/x86/kernel/irq_32.c | 2 +- arch/x86/kernel/kvm.c | 8 +- arch/x86/kernel/kvmclock.c | 2 +- arch/x86/kernel/microcode_core.c | 2 +- arch/x86/kernel/microcode_core_early.c | 6 +- arch/x86/kernel/microcode_intel_early.c | 24 ++--- arch/x86/kernel/mmconf-fam10h_64.c | 12 +-- arch/x86/kernel/msr.c | 4 +- arch/x86/kernel/process.c | 2 +- arch/x86/kernel/setup.c | 2 +- arch/x86/kernel/smpboot.c | 28 +++--- arch/x86/kernel/tboot.c | 4 +- arch/x86/kernel/tsc.c | 4 +- arch/x86/kernel/tsc_sync.c | 18 ++-- arch/x86/kernel/vsyscall_64.c | 6 +- arch/x86/kernel/x86_init.c | 4 +- arch/x86/kernel/xsave.c | 4 +- arch/x86/mm/numa.c | 12 +-- arch/x86/mm/numa_emulation.c | 10 +-- arch/x86/mm/setup_nx.c | 2 +- arch/x86/pci/amd_bus.c | 6 +- arch/x86/platform/ce4100/ce4100.c | 2 +- arch/x86/platform/mrst/mrst.c | 4 +- arch/x86/xen/enlighten.c | 4 +- arch/x86/xen/setup.c | 6 +- arch/x86/xen/smp.c | 12 +-- arch/x86/xen/spinlock.c | 2 +- arch/x86/xen/xen-ops.h | 2 +- arch/xtensa/kernel/time.c | 2 +- block/blk-iopoll.c | 4 +- block/blk-softirq.c | 4 +- drivers/acpi/processor_core.c | 8 +- drivers/acpi/processor_driver.c | 4 +- drivers/acpi/processor_idle.c | 4 +- drivers/base/cpu.c | 4 +- drivers/base/topology.c | 8 +- drivers/clocksource/arm_arch_timer.c | 8 +- drivers/clocksource/exynos_mct.c | 4 +- drivers/clocksource/metag_generic.c | 6 +- drivers/clocksource/time-armada-370-xp.c | 4 +- drivers/clocksource/timer-marco.c | 4 +- drivers/cpufreq/cpufreq_persistent_stats.c | 2 +- drivers/hwmon/coretemp.c | 32 +++---- drivers/hwmon/via-cputemp.c | 6 +- drivers/irqchip/irq-gic.c | 6 +- drivers/oprofile/timer_int.c | 2 +- drivers/xen/xen-acpi-cpuhotplug.c | 2 +- include/linux/cpu.h | 4 +- include/linux/init.h | 12 +-- include/linux/perf_event.h | 2 +- init/calibrate.c | 10 +-- kernel/cpu.c | 6 +- kernel/events/core.c | 4 +- kernel/fork.c | 2 +- kernel/hrtimer.c | 6 +- kernel/printk.c | 6 +- kernel/profile.c | 2 +- kernel/rcutorture.c | 6 +- kernel/rcutree.c | 6 +- kernel/rcutree.h | 4 +- kernel/rcutree_plugin.h | 6 +- kernel/relay.c | 2 +- kernel/sched/core.c | 2 +- kernel/smp.c | 2 +- kernel/smpboot.c | 2 +- kernel/softirq.c | 4 +- kernel/time/tick-sched.c | 2 +- kernel/timer.c | 10 +-- kernel/workqueue.c | 4 +- lib/earlycpio.c | 2 +- lib/percpu_counter.c | 2 +- mm/memcontrol.c | 2 +- mm/page-writeback.c | 4 +- mm/slab.c | 10 +-- mm/slub.c | 4 +- mm/vmstat.c | 6 +- net/core/flow.c | 4 +- net/iucv/iucv.c | 2 +- scripts/mod/modpost.c | 51 +++-------- 303 files changed, 986 insertions(+), 1017 deletions(-) diff --git a/arch/alpha/kernel/smp.c b/arch/alpha/kernel/smp.c index 7b60834fb4b2f..53b18a620e1c2 100644 --- a/arch/alpha/kernel/smp.c +++ b/arch/alpha/kernel/smp.c @@ -116,7 +116,7 @@ wait_boot_cpu_to_stop(int cpuid) /* * Where secondaries begin a life of C. */ -void __cpuinit +void smp_callin(void) { int cpuid = hard_smp_processor_id(); @@ -194,7 +194,7 @@ wait_for_txrdy (unsigned long cpumask) * Send a message to a secondary's console. "START" is one such * interesting message. ;-) */ -static void __cpuinit +static void send_secondary_console_msg(char *str, int cpuid) { struct percpu_struct *cpu; @@ -285,7 +285,7 @@ recv_secondary_console_msg(void) /* * Convince the console to have a secondary cpu begin execution. */ -static int __cpuinit +static int secondary_cpu_start(int cpuid, struct task_struct *idle) { struct percpu_struct *cpu; @@ -356,7 +356,7 @@ secondary_cpu_start(int cpuid, struct task_struct *idle) /* * Bring one cpu online. */ -static int __cpuinit +static int smp_boot_one_cpu(int cpuid, struct task_struct *idle) { unsigned long timeout; @@ -472,7 +472,7 @@ smp_prepare_boot_cpu(void) { } -int __cpuinit +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { smp_boot_one_cpu(cpu, tidle); diff --git a/arch/alpha/kernel/traps.c b/arch/alpha/kernel/traps.c index affccb959a9e8..be1fba334bd07 100644 --- a/arch/alpha/kernel/traps.c +++ b/arch/alpha/kernel/traps.c @@ -32,7 +32,7 @@ static int opDEC_fix; -static void __cpuinit +static void opDEC_check(void) { __asm__ __volatile__ ( @@ -1059,7 +1059,7 @@ do_entUnaUser(void __user * va, unsigned long opcode, return; } -void __cpuinit +void trap_init(void) { /* Tell PAL-code what global pointer we want in the kernel. */ diff --git a/arch/arc/include/asm/irq.h b/arch/arc/include/asm/irq.h index 57898a17eb821..c0a72105ee0b0 100644 --- a/arch/arc/include/asm/irq.h +++ b/arch/arc/include/asm/irq.h @@ -21,6 +21,6 @@ extern void __init arc_init_IRQ(void); extern int __init get_hw_config_num_irq(void); -void __cpuinit arc_local_timer_setup(unsigned int cpu); +void arc_local_timer_setup(unsigned int cpu); #endif diff --git a/arch/arc/kernel/irq.c b/arch/arc/kernel/irq.c index a199471ce01e1..6a9f1bdb36fcf 100644 --- a/arch/arc/kernel/irq.c +++ b/arch/arc/kernel/irq.c @@ -27,7 +27,7 @@ * -Disable all IRQs (on CPU side) * -Optionally, setup the High priority Interrupts as Level 2 IRQs */ -void __cpuinit arc_init_IRQ(void) +void arc_init_IRQ(void) { int level_mask = 0; diff --git a/arch/arc/kernel/setup.c b/arch/arc/kernel/setup.c index 2d7786b69a8a3..eee808381dfe5 100644 --- a/arch/arc/kernel/setup.c +++ b/arch/arc/kernel/setup.c @@ -38,7 +38,7 @@ struct task_struct *_current_task[NR_CPUS]; /* For stack switching */ struct cpuinfo_arc cpuinfo_arc700[NR_CPUS]; -void __cpuinit read_arc_build_cfg_regs(void) +void read_arc_build_cfg_regs(void) { struct bcr_perip uncached_space; struct cpuinfo_arc *cpu = &cpuinfo_arc700[smp_processor_id()]; @@ -234,7 +234,7 @@ char *arc_extn_mumbojumbo(int cpu_id, char *buf, int len) return buf; } -void __cpuinit arc_chk_ccms(void) +void arc_chk_ccms(void) { #if defined(CONFIG_ARC_HAS_DCCM) || defined(CONFIG_ARC_HAS_ICCM) struct cpuinfo_arc *cpu = &cpuinfo_arc700[smp_processor_id()]; @@ -269,7 +269,7 @@ void __cpuinit arc_chk_ccms(void) * hardware has dedicated regs which need to be saved/restored on ctx-sw * (Single Precision uses core regs), thus kernel is kind of oblivious to it */ -void __cpuinit arc_chk_fpu(void) +void arc_chk_fpu(void) { struct cpuinfo_arc *cpu = &cpuinfo_arc700[smp_processor_id()]; @@ -290,7 +290,7 @@ void __cpuinit arc_chk_fpu(void) * such as only for boot CPU etc */ -void __cpuinit setup_processor(void) +void setup_processor(void) { char str[512]; int cpu_id = smp_processor_id(); diff --git a/arch/arc/kernel/smp.c b/arch/arc/kernel/smp.c index 5c7fd603d216c..bca3052c956da 100644 --- a/arch/arc/kernel/smp.c +++ b/arch/arc/kernel/smp.c @@ -117,7 +117,7 @@ const char *arc_platform_smp_cpuinfo(void) * Called from asm stub in head.S * "current"/R25 already setup by low level boot code */ -void __cpuinit start_kernel_secondary(void) +void start_kernel_secondary(void) { struct mm_struct *mm = &init_mm; unsigned int cpu = smp_processor_id(); @@ -154,7 +154,7 @@ void __cpuinit start_kernel_secondary(void) * * Essential requirements being where to run from (PC) and stack (SP) */ -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) +int __cpu_up(unsigned int cpu, struct task_struct *idle) { unsigned long wait_till; diff --git a/arch/arc/kernel/time.c b/arch/arc/kernel/time.c index 09f4309aa2c03..c8bb48476c296 100644 --- a/arch/arc/kernel/time.c +++ b/arch/arc/kernel/time.c @@ -50,7 +50,7 @@ #ifdef CONFIG_ARC_HAS_RTSC -int __cpuinit arc_counter_setup(void) +int arc_counter_setup(void) { /* RTSC insn taps into cpu clk, needs no setup */ @@ -105,7 +105,7 @@ static bool is_usable_as_clocksource(void) /* * set 32bit TIMER1 to keep counting monotonically and wraparound */ -int __cpuinit arc_counter_setup(void) +int arc_counter_setup(void) { write_aux_reg(ARC_REG_TIMER1_LIMIT, ARC_TIMER_MAX); write_aux_reg(ARC_REG_TIMER1_CNT, 0); @@ -212,7 +212,7 @@ static struct irqaction arc_timer_irq = { * Setup the local event timer for @cpu * N.B. weak so that some exotic ARC SoCs can completely override it */ -void __attribute__((weak)) __cpuinit arc_local_timer_setup(unsigned int cpu) +void __attribute__((weak)) arc_local_timer_setup(unsigned int cpu) { struct clock_event_device *clk = &per_cpu(arc_clockevent_device, cpu); diff --git a/arch/arc/mm/cache_arc700.c b/arch/arc/mm/cache_arc700.c index aedce1905441c..eebbd7bf14986 100644 --- a/arch/arc/mm/cache_arc700.c +++ b/arch/arc/mm/cache_arc700.c @@ -100,7 +100,7 @@ char *arc_cache_mumbojumbo(int cpu_id, char *buf, int len) * the cpuinfo structure for later use. * No Validation done here, simply read/convert the BCRs */ -void __cpuinit read_decode_cache_bcr(void) +void read_decode_cache_bcr(void) { struct bcr_cache ibcr, dbcr; struct cpuinfo_arc_cache *p_ic, *p_dc; @@ -132,7 +132,7 @@ void __cpuinit read_decode_cache_bcr(void) * 3. Enable the Caches, setup default flush mode for D-Cache * 3. Calculate the SHMLBA used by user space */ -void __cpuinit arc_cache_init(void) +void arc_cache_init(void) { unsigned int temp; unsigned int cpu = smp_processor_id(); diff --git a/arch/arc/mm/tlb.c b/arch/arc/mm/tlb.c index fe1c5a073afe4..43abf8dbc07fa 100644 --- a/arch/arc/mm/tlb.c +++ b/arch/arc/mm/tlb.c @@ -464,7 +464,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long vaddr_unaligned, * the cpuinfo structure for later use. * No Validation is done here, simply read/convert the BCRs */ -void __cpuinit read_decode_mmu_bcr(void) +void read_decode_mmu_bcr(void) { unsigned int tmp; struct bcr_mmu_1_2 *mmu2; /* encoded MMU2 attr */ @@ -510,7 +510,7 @@ char *arc_mmu_mumbojumbo(int cpu_id, char *buf, int len) return buf; } -void __cpuinit arc_mmu_init(void) +void arc_mmu_init(void) { char str[256]; struct cpuinfo_arc_mmu *mmu = &cpuinfo_arc700[smp_processor_id()].mmu; diff --git a/arch/arm/common/mcpm_platsmp.c b/arch/arm/common/mcpm_platsmp.c index 3caed0db69861..79892a52234a6 100644 --- a/arch/arm/common/mcpm_platsmp.c +++ b/arch/arm/common/mcpm_platsmp.c @@ -23,7 +23,7 @@ static void __init simple_smp_init_cpus(void) { } -static int __cpuinit mcpm_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int mcpm_boot_secondary(unsigned int cpu, struct task_struct *idle) { unsigned int mpidr, pcpu, pcluster, ret; extern void secondary_startup(void); @@ -44,7 +44,7 @@ static int __cpuinit mcpm_boot_secondary(unsigned int cpu, struct task_struct *i return 0; } -static void __cpuinit mcpm_secondary_init(unsigned int cpu) +static void mcpm_secondary_init(unsigned int cpu) { mcpm_cpu_powered_up(); } diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index 1122ba9dce3df..fb52db4e076ff 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -1042,7 +1042,7 @@ static void reset_ctrl_regs(void *unused) cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu)); } -static int __cpuinit dbg_reset_notify(struct notifier_block *self, +static int dbg_reset_notify(struct notifier_block *self, unsigned long action, void *cpu) { if ((action & ~CPU_TASKS_FROZEN) == CPU_ONLINE) @@ -1051,7 +1051,7 @@ static int __cpuinit dbg_reset_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata dbg_reset_nb = { +static struct notifier_block dbg_reset_nb = { .notifier_call = dbg_reset_notify, }; diff --git a/arch/arm/kernel/perf_event_cpu.c b/arch/arm/kernel/perf_event_cpu.c index 34b363c09202b..e4479c7903aa7 100644 --- a/arch/arm/kernel/perf_event_cpu.c +++ b/arch/arm/kernel/perf_event_cpu.c @@ -257,7 +257,7 @@ static void armpmu_hotplug_disable(void *parm_pmu) * UNKNOWN at reset, the PMU must be explicitly reset to avoid reading * junk values out of them. */ -static int __cpuinit cpu_pmu_notify(struct notifier_block *b, +static int cpu_pmu_notify(struct notifier_block *b, unsigned long action, void *hcpu) { int irq; @@ -317,7 +317,7 @@ static int __cpuinit cpu_pmu_notify(struct notifier_block *b, return ret; } -static struct notifier_block __cpuinitdata cpu_pmu_hotplug_notifier = { +static struct notifier_block cpu_pmu_hotplug_notifier = { .notifier_call = cpu_pmu_notify, }; diff --git a/arch/arm/kernel/psci_smp.c b/arch/arm/kernel/psci_smp.c index fc1dbd2582552..da343ee426499 100644 --- a/arch/arm/kernel/psci_smp.c +++ b/arch/arm/kernel/psci_smp.c @@ -47,7 +47,7 @@ extern void secondary_startup(void); -static int __cpuinit psci_boot_secondary(unsigned int cpu, +static int psci_boot_secondary(unsigned int cpu, struct task_struct *idle) { if (psci_ops.cpu_on) diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index a471f654463bb..e242f129537fd 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -57,7 +57,7 @@ struct secondary_data secondary_data; * control for which core is the next to come out of the secondary * boot "holding pen" */ -volatile int __cpuinitdata pen_release = -1; +volatile int pen_release = -1; enum ipi_msg_type { IPI_WAKEUP, @@ -79,7 +79,7 @@ void __init smp_set_ops(struct smp_operations *ops) smp_ops = *ops; }; -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) +int __cpu_up(unsigned int cpu, struct task_struct *idle) { int ret; @@ -126,7 +126,7 @@ void __init smp_init_cpus(void) smp_ops.smp_init_cpus(); } -int __cpuinit boot_secondary(unsigned int cpu, struct task_struct *idle) +int boot_secondary(unsigned int cpu, struct task_struct *idle) { if (smp_ops.smp_boot_secondary) return smp_ops.smp_boot_secondary(cpu, idle); @@ -158,7 +158,7 @@ static int platform_cpu_disable(unsigned int cpu) /* * __cpu_disable runs on the processor to be shutdown. */ -int __cpuinit __cpu_disable(void) +int __cpu_disable(void) { unsigned int cpu = smp_processor_id(); int ret; @@ -204,7 +204,7 @@ static DECLARE_COMPLETION(cpu_died); * called on the thread which is asking for a CPU to be shutdown - * waits until shutdown has completed, or it is timed out. */ -void __cpuinit __cpu_die(unsigned int cpu) +void __cpu_die(unsigned int cpu) { if (!wait_for_completion_timeout(&cpu_died, msecs_to_jiffies(5000))) { pr_err("CPU%u: cpu didn't die\n", cpu); @@ -294,7 +294,7 @@ void __ref cpu_die(void) * Called by both boot and secondaries to move global data into * per-processor storage. */ -static void __cpuinit smp_store_cpu_info(unsigned int cpuid) +static void smp_store_cpu_info(unsigned int cpuid) { struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid); @@ -310,7 +310,7 @@ static void percpu_timer_setup(void); * This is the secondary CPU boot entry. We're using this CPUs * idle thread stack, but a set of temporary page tables. */ -asmlinkage void __cpuinit secondary_start_kernel(void) +asmlinkage void secondary_start_kernel(void) { struct mm_struct *mm = &init_mm; unsigned int cpu; @@ -522,7 +522,7 @@ static void broadcast_timer_set_mode(enum clock_event_mode mode, { } -static void __cpuinit broadcast_timer_setup(struct clock_event_device *evt) +static void broadcast_timer_setup(struct clock_event_device *evt) { evt->name = "dummy_timer"; evt->features = CLOCK_EVT_FEAT_ONESHOT | @@ -551,7 +551,7 @@ int local_timer_register(struct local_timer_ops *ops) } #endif -static void __cpuinit percpu_timer_setup(void) +static void percpu_timer_setup(void) { unsigned int cpu = smp_processor_id(); struct clock_event_device *evt = &per_cpu(percpu_clockevent, cpu); diff --git a/arch/arm/kernel/smp_twd.c b/arch/arm/kernel/smp_twd.c index f6fd1d4398c6f..25956204ef23e 100644 --- a/arch/arm/kernel/smp_twd.c +++ b/arch/arm/kernel/smp_twd.c @@ -187,7 +187,7 @@ core_initcall(twd_cpufreq_init); #endif -static void __cpuinit twd_calibrate_rate(void) +static void twd_calibrate_rate(void) { unsigned long count; u64 waitjiffies; @@ -265,7 +265,7 @@ static void twd_get_clock(struct device_node *np) /* * Setup the local clock events for a CPU. */ -static int __cpuinit twd_timer_setup(struct clock_event_device *clk) +static int twd_timer_setup(struct clock_event_device *clk) { struct clock_event_device **this_cpu_clk; int cpu = smp_processor_id(); @@ -308,7 +308,7 @@ static int __cpuinit twd_timer_setup(struct clock_event_device *clk) return 0; } -static struct local_timer_ops twd_lt_ops __cpuinitdata = { +static struct local_timer_ops twd_lt_ops = { .setup = twd_timer_setup, .stop = twd_timer_stop, }; diff --git a/arch/arm/lib/delay.c b/arch/arm/lib/delay.c index 64dbfa57204ae..5306de3501339 100644 --- a/arch/arm/lib/delay.c +++ b/arch/arm/lib/delay.c @@ -86,7 +86,7 @@ void __init register_current_timer_delay(const struct delay_timer *timer) } } -unsigned long __cpuinit calibrate_delay_is_known(void) +unsigned long calibrate_delay_is_known(void) { delay_calibrated = true; return lpj_fine; diff --git a/arch/arm/mach-exynos/platsmp.c b/arch/arm/mach-exynos/platsmp.c index a0e8ff7758a4c..01442f1ee16d1 100644 --- a/arch/arm/mach-exynos/platsmp.c +++ b/arch/arm/mach-exynos/platsmp.c @@ -73,7 +73,7 @@ static void __iomem *scu_base_addr(void) static DEFINE_SPINLOCK(boot_lock); -static void __cpuinit exynos_secondary_init(unsigned int cpu) +static void exynos_secondary_init(unsigned int cpu) { /* * let the primary processor know we're out of the @@ -88,7 +88,7 @@ static void __cpuinit exynos_secondary_init(unsigned int cpu) spin_unlock(&boot_lock); } -static int __cpuinit exynos_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int exynos_boot_secondary(unsigned int cpu, struct task_struct *idle) { unsigned long timeout; unsigned long phys_cpu = cpu_logical_map(cpu); diff --git a/arch/arm/mach-highbank/platsmp.c b/arch/arm/mach-highbank/platsmp.c index a984573e0d023..32d75cf55cbc7 100644 --- a/arch/arm/mach-highbank/platsmp.c +++ b/arch/arm/mach-highbank/platsmp.c @@ -24,7 +24,7 @@ extern void secondary_startup(void); -static int __cpuinit highbank_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int highbank_boot_secondary(unsigned int cpu, struct task_struct *idle) { highbank_set_cpu_jump(cpu, secondary_startup); arch_send_wakeup_ipi_mask(cpumask_of(cpu)); diff --git a/arch/arm/mach-imx/platsmp.c b/arch/arm/mach-imx/platsmp.c index c6e1ab5448822..1f24c1fdfea4e 100644 --- a/arch/arm/mach-imx/platsmp.c +++ b/arch/arm/mach-imx/platsmp.c @@ -53,7 +53,7 @@ void imx_scu_standby_enable(void) writel_relaxed(val, scu_base); } -static int __cpuinit imx_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int imx_boot_secondary(unsigned int cpu, struct task_struct *idle) { imx_set_cpu_jump(cpu, v7_secondary_startup); imx_enable_cpu(cpu, true); diff --git a/arch/arm/mach-msm/platsmp.c b/arch/arm/mach-msm/platsmp.c index f4f29e4f40045..ba758992e862b 100644 --- a/arch/arm/mach-msm/platsmp.c +++ b/arch/arm/mach-msm/platsmp.c @@ -42,7 +42,7 @@ * observers, irrespective of whether they're taking part in coherency * or not. This is necessary for the hotplug code to work reliably. */ -void __cpuinit write_pen_release(int val) +void write_pen_release(int val) { pen_release = val; smp_wmb(); @@ -52,7 +52,7 @@ void __cpuinit write_pen_release(int val) static DEFINE_SPINLOCK(boot_lock); -void __cpuinit msm_secondary_init(unsigned int cpu) +void msm_secondary_init(unsigned int cpu) { WARN_ON(msm_platform_secondary_init(cpu)); @@ -69,7 +69,7 @@ void __cpuinit msm_secondary_init(unsigned int cpu) spin_unlock(&boot_lock); } -static int __cpuinit release_secondary_sim(unsigned long base, unsigned int cpu) +static int release_secondary_sim(unsigned long base, unsigned int cpu) { void *base_ptr = ioremap_nocache(base + (cpu * 0x10000), SZ_4K); if (!base_ptr) @@ -83,7 +83,7 @@ static int __cpuinit release_secondary_sim(unsigned long base, unsigned int cpu) return 0; } -static int __cpuinit scorpion_release_secondary(void) +static int scorpion_release_secondary(void) { void *base_ptr = ioremap_nocache(0x00902000, SZ_4K*2); if (!base_ptr) @@ -99,7 +99,7 @@ static int __cpuinit scorpion_release_secondary(void) return 0; } -static int __cpuinit msm8960_release_secondary(unsigned long base, +static int msm8960_release_secondary(unsigned long base, unsigned int cpu) { void *base_ptr = ioremap_nocache(base + (cpu * 0x10000), SZ_4K); @@ -129,7 +129,7 @@ static int __cpuinit msm8960_release_secondary(unsigned long base, return 0; } -static int __cpuinit msm8974_release_secondary(unsigned long base, +static int msm8974_release_secondary(unsigned long base, unsigned int cpu) { void *base_ptr = ioremap_nocache(base + (cpu * 0x10000), SZ_4K); @@ -156,7 +156,7 @@ static int __cpuinit msm8974_release_secondary(unsigned long base, return 0; } -static int __cpuinit arm_release_secondary(unsigned long base, unsigned int cpu) +static int arm_release_secondary(unsigned long base, unsigned int cpu) { void *base_ptr = ioremap_nocache(base + (cpu * 0x10000), SZ_4K); if (!base_ptr) @@ -191,7 +191,7 @@ static int __cpuinit arm_release_secondary(unsigned long base, unsigned int cpu) return 0; } -static int __cpuinit release_from_pen(unsigned int cpu) +static int release_from_pen(unsigned int cpu) { unsigned long timeout; @@ -241,7 +241,7 @@ static int __cpuinit release_from_pen(unsigned int cpu) DEFINE_PER_CPU(int, cold_boot_done); -int __cpuinit scorpion_boot_secondary(unsigned int cpu, +int scorpion_boot_secondary(unsigned int cpu, struct task_struct *idle) { pr_debug("Starting secondary CPU %d\n", cpu); @@ -253,7 +253,7 @@ int __cpuinit scorpion_boot_secondary(unsigned int cpu, return release_from_pen(cpu); } -int __cpuinit msm8960_boot_secondary(unsigned int cpu, struct task_struct *idle) +int msm8960_boot_secondary(unsigned int cpu, struct task_struct *idle) { pr_debug("Starting secondary CPU %d\n", cpu); @@ -264,7 +264,7 @@ int __cpuinit msm8960_boot_secondary(unsigned int cpu, struct task_struct *idle) return release_from_pen(cpu); } -int __cpuinit msm8974_boot_secondary(unsigned int cpu, struct task_struct *idle) +int msm8974_boot_secondary(unsigned int cpu, struct task_struct *idle) { pr_debug("Starting secondary CPU %d\n", cpu); @@ -279,7 +279,7 @@ int __cpuinit msm8974_boot_secondary(unsigned int cpu, struct task_struct *idle) return release_from_pen(cpu); } -static int __cpuinit msm8916_boot_secondary(unsigned int cpu, +static int msm8916_boot_secondary(unsigned int cpu, struct task_struct *idle) { pr_debug("Starting secondary CPU %d\n", cpu); @@ -295,7 +295,7 @@ static int __cpuinit msm8916_boot_secondary(unsigned int cpu, return release_from_pen(cpu); } -static int __cpuinit msm8936_boot_secondary(unsigned int cpu, +static int msm8936_boot_secondary(unsigned int cpu, struct task_struct *idle) { int ret = 0; @@ -317,7 +317,7 @@ static int __cpuinit msm8936_boot_secondary(unsigned int cpu, return release_from_pen(cpu); } -int __cpuinit arm_boot_secondary(unsigned int cpu, struct task_struct *idle) +int arm_boot_secondary(unsigned int cpu, struct task_struct *idle) { pr_debug("Starting secondary CPU %d\n", cpu); diff --git a/arch/arm/mach-msm/timer.c b/arch/arm/mach-msm/timer.c index d749417c473d6..23d74cc68b952 100644 --- a/arch/arm/mach-msm/timer.c +++ b/arch/arm/mach-msm/timer.c @@ -889,7 +889,7 @@ static void __init msm_sched_clock_init(void) } #ifdef CONFIG_LOCAL_TIMERS -int __cpuinit local_timer_setup(struct clock_event_device *evt) +int local_timer_setup(struct clock_event_device *evt) { static DEFINE_PER_CPU(bool, first_boot) = true; struct msm_clock *clock = &msm_clocks[msm_global_timer]; diff --git a/arch/arm/mach-mvebu/platsmp.c b/arch/arm/mach-mvebu/platsmp.c index 875ea748391ca..703f9ccf435f5 100644 --- a/arch/arm/mach-mvebu/platsmp.c +++ b/arch/arm/mach-mvebu/platsmp.c @@ -71,12 +71,12 @@ void __init set_secondary_cpus_clock(void) } } -static void __cpuinit armada_xp_secondary_init(unsigned int cpu) +static void armada_xp_secondary_init(unsigned int cpu) { armada_xp_mpic_smp_cpu_init(); } -static int __cpuinit armada_xp_boot_secondary(unsigned int cpu, +static int armada_xp_boot_secondary(unsigned int cpu, struct task_struct *idle) { pr_info("Booting CPU %d\n", cpu); diff --git a/arch/arm/mach-omap2/omap-mpuss-lowpower.c b/arch/arm/mach-omap2/omap-mpuss-lowpower.c index e80327b6c81f6..1576ed36021d5 100644 --- a/arch/arm/mach-omap2/omap-mpuss-lowpower.c +++ b/arch/arm/mach-omap2/omap-mpuss-lowpower.c @@ -254,7 +254,7 @@ int omap4_enter_lowpower(unsigned int cpu, unsigned int power_state) * @cpu : CPU ID * @power_state: CPU low power state. */ -int __cpuinit omap4_hotplug_cpu(unsigned int cpu, unsigned int power_state) +int omap4_hotplug_cpu(unsigned int cpu, unsigned int power_state) { struct omap4_cpu_pm_info *pm_info = &per_cpu(omap4_pm_info, cpu); unsigned int cpu_state = 0; diff --git a/arch/arm/mach-omap2/omap-smp.c b/arch/arm/mach-omap2/omap-smp.c index 2a551f997aea5..818812bed623a 100644 --- a/arch/arm/mach-omap2/omap-smp.c +++ b/arch/arm/mach-omap2/omap-smp.c @@ -51,7 +51,7 @@ void __iomem *omap4_get_scu_base(void) return scu_base; } -static void __cpuinit omap4_secondary_init(unsigned int cpu) +static void omap4_secondary_init(unsigned int cpu) { /* * Configure ACTRL and enable NS SMP bit access on CPU1 on HS device. @@ -72,7 +72,7 @@ static void __cpuinit omap4_secondary_init(unsigned int cpu) spin_unlock(&boot_lock); } -static int __cpuinit omap4_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int omap4_boot_secondary(unsigned int cpu, struct task_struct *idle) { static struct clockdomain *cpu1_clkdm; static bool booted; diff --git a/arch/arm/mach-omap2/omap-wakeupgen.c b/arch/arm/mach-omap2/omap-wakeupgen.c index f8bb3b9b6a762..ed4b7d3bd5da5 100644 --- a/arch/arm/mach-omap2/omap-wakeupgen.c +++ b/arch/arm/mach-omap2/omap-wakeupgen.c @@ -323,7 +323,7 @@ static void irq_save_secure_context(void) #endif #ifdef CONFIG_HOTPLUG_CPU -static int __cpuinit irq_cpu_hotplug_notify(struct notifier_block *self, +static int irq_cpu_hotplug_notify(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned int)hcpu; diff --git a/arch/arm/mach-prima2/platsmp.c b/arch/arm/mach-prima2/platsmp.c index 1c3de7bed8413..3dbcb1ab6e37e 100644 --- a/arch/arm/mach-prima2/platsmp.c +++ b/arch/arm/mach-prima2/platsmp.c @@ -44,7 +44,7 @@ void __init sirfsoc_map_scu(void) scu_base = (void __iomem *)SIRFSOC_VA(base); } -static void __cpuinit sirfsoc_secondary_init(unsigned int cpu) +static void sirfsoc_secondary_init(unsigned int cpu) { /* * let the primary processor know we're out of the @@ -65,7 +65,7 @@ static struct of_device_id rsc_ids[] = { {}, }; -static int __cpuinit sirfsoc_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int sirfsoc_boot_secondary(unsigned int cpu, struct task_struct *idle) { unsigned long timeout; struct device_node *np; diff --git a/arch/arm/mach-shmobile/smp-emev2.c b/arch/arm/mach-shmobile/smp-emev2.c index e38691b4d0ddc..ee58c718752d1 100644 --- a/arch/arm/mach-shmobile/smp-emev2.c +++ b/arch/arm/mach-shmobile/smp-emev2.c @@ -30,7 +30,7 @@ #define EMEV2_SCU_BASE 0x1e000000 -static int __cpuinit emev2_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int emev2_boot_secondary(unsigned int cpu, struct task_struct *idle) { arch_send_wakeup_ipi_mask(cpumask_of(cpu_logical_map(cpu))); return 0; diff --git a/arch/arm/mach-shmobile/smp-r8a7779.c b/arch/arm/mach-shmobile/smp-r8a7779.c index a853bf182ed5e..77ee71ab2f1d2 100644 --- a/arch/arm/mach-shmobile/smp-r8a7779.c +++ b/arch/arm/mach-shmobile/smp-r8a7779.c @@ -81,7 +81,7 @@ static int r8a7779_platform_cpu_kill(unsigned int cpu) return ret ? ret : 1; } -static int __cpuinit r8a7779_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int r8a7779_boot_secondary(unsigned int cpu, struct task_struct *idle) { struct r8a7779_pm_ch *ch = NULL; int ret = -EIO; diff --git a/arch/arm/mach-shmobile/smp-sh73a0.c b/arch/arm/mach-shmobile/smp-sh73a0.c index 496592b6c7639..91584799e7056 100644 --- a/arch/arm/mach-shmobile/smp-sh73a0.c +++ b/arch/arm/mach-shmobile/smp-sh73a0.c @@ -48,7 +48,7 @@ void __init sh73a0_register_twd(void) } #endif -static int __cpuinit sh73a0_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int sh73a0_boot_secondary(unsigned int cpu, struct task_struct *idle) { cpu = cpu_logical_map(cpu); diff --git a/arch/arm/mach-socfpga/platsmp.c b/arch/arm/mach-socfpga/platsmp.c index b51ce8c7929dc..5356a72bc8ce9 100644 --- a/arch/arm/mach-socfpga/platsmp.c +++ b/arch/arm/mach-socfpga/platsmp.c @@ -29,7 +29,7 @@ #include "core.h" -static int __cpuinit socfpga_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int socfpga_boot_secondary(unsigned int cpu, struct task_struct *idle) { int trampoline_size = &secondary_trampoline_end - &secondary_trampoline; diff --git a/arch/arm/mach-spear/generic.h b/arch/arm/mach-spear/generic.h index 904f2c907b46d..a99d90a4d09c5 100644 --- a/arch/arm/mach-spear/generic.h +++ b/arch/arm/mach-spear/generic.h @@ -37,7 +37,7 @@ void __init spear13xx_l2x0_init(void); void spear_restart(enum reboot_mode, const char *); void spear13xx_secondary_startup(void); -void __cpuinit spear13xx_cpu_die(unsigned int cpu); +void spear13xx_cpu_die(unsigned int cpu); extern struct smp_operations spear13xx_smp_ops; diff --git a/arch/arm/mach-spear/platsmp.c b/arch/arm/mach-spear/platsmp.c index 9c4c722c954e0..5c4a19887b2bb 100644 --- a/arch/arm/mach-spear/platsmp.c +++ b/arch/arm/mach-spear/platsmp.c @@ -24,7 +24,7 @@ static DEFINE_SPINLOCK(boot_lock); static void __iomem *scu_base = IOMEM(VA_SCU_BASE); -static void __cpuinit spear13xx_secondary_init(unsigned int cpu) +static void spear13xx_secondary_init(unsigned int cpu) { /* * let the primary processor know we're out of the @@ -40,7 +40,7 @@ static void __cpuinit spear13xx_secondary_init(unsigned int cpu) spin_unlock(&boot_lock); } -static int __cpuinit spear13xx_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int spear13xx_boot_secondary(unsigned int cpu, struct task_struct *idle) { unsigned long timeout; diff --git a/arch/arm/mach-tegra/platsmp.c b/arch/arm/mach-tegra/platsmp.c index fad4226ef710d..8aa84b50a4bba 100644 --- a/arch/arm/mach-tegra/platsmp.c +++ b/arch/arm/mach-tegra/platsmp.c @@ -35,7 +35,7 @@ static cpumask_t tegra_cpu_init_mask; -static void __cpuinit tegra_secondary_init(unsigned int cpu) +static void tegra_secondary_init(unsigned int cpu) { cpumask_set_cpu(cpu, &tegra_cpu_init_mask); } @@ -144,7 +144,7 @@ static int tegra114_boot_secondary(unsigned int cpu, struct task_struct *idle) return tegra_pmc_cpu_power_on(cpu); } -static int __cpuinit tegra_boot_secondary(unsigned int cpu, +static int tegra_boot_secondary(unsigned int cpu, struct task_struct *idle) { if (IS_ENABLED(CONFIG_ARCH_TEGRA_2x_SOC) && tegra_chip_id == TEGRA20) diff --git a/arch/arm/mach-tegra/pm.c b/arch/arm/mach-tegra/pm.c index e7594fa85066a..9718125a68a5b 100644 --- a/arch/arm/mach-tegra/pm.c +++ b/arch/arm/mach-tegra/pm.c @@ -175,7 +175,7 @@ static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = { [TEGRA_SUSPEND_LP0] = "LP0", }; -static int __cpuinit tegra_suspend_enter(suspend_state_t state) +static int tegra_suspend_enter(suspend_state_t state) { enum tegra_suspend_mode mode = tegra_pmc_get_suspend_mode(); diff --git a/arch/arm/mach-ux500/platsmp.c b/arch/arm/mach-ux500/platsmp.c index 14d90469392f1..1f296e796a4fe 100644 --- a/arch/arm/mach-ux500/platsmp.c +++ b/arch/arm/mach-ux500/platsmp.c @@ -54,7 +54,7 @@ static void __iomem *scu_base_addr(void) static DEFINE_SPINLOCK(boot_lock); -static void __cpuinit ux500_secondary_init(unsigned int cpu) +static void ux500_secondary_init(unsigned int cpu) { /* * let the primary processor know we're out of the @@ -69,7 +69,7 @@ static void __cpuinit ux500_secondary_init(unsigned int cpu) spin_unlock(&boot_lock); } -static int __cpuinit ux500_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int ux500_boot_secondary(unsigned int cpu, struct task_struct *idle) { unsigned long timeout; diff --git a/arch/arm/mach-zynq/common.h b/arch/arm/mach-zynq/common.h index fbbd0e21c404d..3040d219570f2 100644 --- a/arch/arm/mach-zynq/common.h +++ b/arch/arm/mach-zynq/common.h @@ -27,7 +27,7 @@ extern void secondary_startup(void); extern char zynq_secondary_trampoline; extern char zynq_secondary_trampoline_jump; extern char zynq_secondary_trampoline_end; -extern int __cpuinit zynq_cpun_start(u32 address, int cpu); +extern int zynq_cpun_start(u32 address, int cpu); extern struct smp_operations zynq_smp_ops __initdata; #endif diff --git a/arch/arm/mach-zynq/platsmp.c b/arch/arm/mach-zynq/platsmp.c index 5fc167e076199..6bc2270b35704 100644 --- a/arch/arm/mach-zynq/platsmp.c +++ b/arch/arm/mach-zynq/platsmp.c @@ -30,11 +30,11 @@ /* * Store number of cores in the system * Because of scu_get_core_count() must be in __init section and can't - * be called from zynq_cpun_start() because it is in __cpuinit section. + * be called from zynq_cpun_start() because it is in section. */ static int ncores; -int __cpuinit zynq_cpun_start(u32 address, int cpu) +int zynq_cpun_start(u32 address, int cpu) { u32 trampoline_code_size = &zynq_secondary_trampoline_end - &zynq_secondary_trampoline; @@ -92,7 +92,7 @@ int __cpuinit zynq_cpun_start(u32 address, int cpu) } EXPORT_SYMBOL(zynq_cpun_start); -static int __cpuinit zynq_boot_secondary(unsigned int cpu, +static int zynq_boot_secondary(unsigned int cpu, struct task_struct *idle) { return zynq_cpun_start(virt_to_phys(secondary_startup), cpu); diff --git a/arch/arm/plat-versatile/platsmp.c b/arch/arm/plat-versatile/platsmp.c index 1e1b2d7697488..39895d892c3be 100644 --- a/arch/arm/plat-versatile/platsmp.c +++ b/arch/arm/plat-versatile/platsmp.c @@ -23,7 +23,7 @@ * observers, irrespective of whether they're taking part in coherency * or not. This is necessary for the hotplug code to work reliably. */ -static void __cpuinit write_pen_release(int val) +static void write_pen_release(int val) { pen_release = val; smp_wmb(); @@ -33,7 +33,7 @@ static void __cpuinit write_pen_release(int val) static DEFINE_SPINLOCK(boot_lock); -void __cpuinit versatile_secondary_init(unsigned int cpu) +void versatile_secondary_init(unsigned int cpu) { /* * let the primary processor know we're out of the @@ -48,7 +48,7 @@ void __cpuinit versatile_secondary_init(unsigned int cpu) spin_unlock(&boot_lock); } -int __cpuinit versatile_boot_secondary(unsigned int cpu, struct task_struct *idle) +int versatile_boot_secondary(unsigned int cpu, struct task_struct *idle) { unsigned long timeout; diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index bb1878871b9a8..2eaec55fcfbf2 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -139,7 +139,7 @@ static void clear_os_lock(void *unused) asm volatile("msr oslar_el1, %0" : : "r" (0)); } -static int __cpuinit os_lock_notify(struct notifier_block *self, +static int os_lock_notify(struct notifier_block *self, unsigned long action, void *data) { int cpu = (unsigned long)data; @@ -148,11 +148,11 @@ static int __cpuinit os_lock_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata os_lock_nb = { +static struct notifier_block os_lock_nb = { .notifier_call = os_lock_notify, }; -static int __cpuinit debug_monitors_init(void) +static int debug_monitors_init(void) { cpu_notifier_register_begin(); diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c index de7b854438d79..db3b6fd716bc6 100644 --- a/arch/arm64/kernel/hw_breakpoint.c +++ b/arch/arm64/kernel/hw_breakpoint.c @@ -880,7 +880,7 @@ static void hw_breakpoint_reset(void *unused) } } -static int __cpuinit hw_breakpoint_reset_notify(struct notifier_block *self, +static int hw_breakpoint_reset_notify(struct notifier_block *self, unsigned long action, void *hcpu) { @@ -890,7 +890,7 @@ static int __cpuinit hw_breakpoint_reset_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata hw_breakpoint_reset_nb = { +static struct notifier_block hw_breakpoint_reset_nb = { .notifier_call = hw_breakpoint_reset_notify, }; diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c index 21f3b11136bed..fc77c1e4c656e 100644 --- a/arch/arm64/kernel/perf_event.c +++ b/arch/arm64/kernel/perf_event.c @@ -1519,8 +1519,8 @@ static void armpmu_hotplug_disable(void *parm_pmu) * UNKNOWN at reset, the PMU must be explicitly reset to avoid reading * junk values out of them. */ -static int __cpuinit cpu_pmu_notify(struct notifier_block *b, - unsigned long action, void *hcpu) +static int cpu_pmu_notify(struct notifier_block *b, + unsigned long action, void *hcpu) { int irq; struct pmu *pmu; diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 73ad727ccd566..9c7c1545fd494 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -74,7 +74,7 @@ enum ipi_msg_type { * Boot a secondary CPU, and assign it the specified idle task. * This also gives us the initial stack to use for this CPU. */ -static int __cpuinit boot_secondary(unsigned int cpu, struct task_struct *idle) +static int boot_secondary(unsigned int cpu, struct task_struct *idle) { if (cpu_ops[cpu]->cpu_boot) return cpu_ops[cpu]->cpu_boot(cpu); @@ -84,7 +84,7 @@ static int __cpuinit boot_secondary(unsigned int cpu, struct task_struct *idle) static DECLARE_COMPLETION(cpu_running); -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) +int __cpu_up(unsigned int cpu, struct task_struct *idle) { int ret; @@ -129,7 +129,7 @@ static void smp_store_cpu_info(unsigned int cpuid) * This is the secondary CPU boot entry. We're using this CPUs * idle thread stack, but a set of temporary page tables. */ -asmlinkage void __cpuinit secondary_start_kernel(void) +asmlinkage void secondary_start_kernel(void) { struct mm_struct *mm = &init_mm; unsigned int cpu = smp_processor_id(); diff --git a/arch/blackfin/kernel/perf_event.c b/arch/blackfin/kernel/perf_event.c index e47d19ae3e063..974e55496db30 100644 --- a/arch/blackfin/kernel/perf_event.c +++ b/arch/blackfin/kernel/perf_event.c @@ -468,7 +468,7 @@ static void bfin_pmu_setup(int cpu) memset(cpuhw, 0, sizeof(struct cpu_hw_events)); } -static int __cpuinit +static int bfin_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (long)hcpu; diff --git a/arch/blackfin/kernel/setup.c b/arch/blackfin/kernel/setup.c index 107b306b06f10..19ad0637e8ff0 100644 --- a/arch/blackfin/kernel/setup.c +++ b/arch/blackfin/kernel/setup.c @@ -99,7 +99,7 @@ void __init generate_cplb_tables(void) } #endif -void __cpuinit bfin_setup_caches(unsigned int cpu) +void bfin_setup_caches(unsigned int cpu) { #ifdef CONFIG_BFIN_ICACHE bfin_icache_init(icplb_tbl[cpu]); @@ -165,7 +165,7 @@ void __cpuinit bfin_setup_caches(unsigned int cpu) #endif } -void __cpuinit bfin_setup_cpudata(unsigned int cpu) +void bfin_setup_cpudata(unsigned int cpu) { struct blackfin_cpudata *cpudata = &per_cpu(cpu_data, cpu); diff --git a/arch/blackfin/mach-bf561/smp.c b/arch/blackfin/mach-bf561/smp.c index ab1c617b9cfc9..7748b84f1e541 100644 --- a/arch/blackfin/mach-bf561/smp.c +++ b/arch/blackfin/mach-bf561/smp.c @@ -48,7 +48,7 @@ int __init setup_profiling_timer(unsigned int multiplier) /* not supported */ return -EINVAL; } -void __cpuinit platform_secondary_init(unsigned int cpu) +void platform_secondary_init(unsigned int cpu) { /* Clone setup for peripheral interrupt sources from CoreA. */ bfin_write_SICB_IMASK0(bfin_read_SIC_IMASK0()); @@ -74,7 +74,7 @@ void __cpuinit platform_secondary_init(unsigned int cpu) spin_unlock(&boot_lock); } -int __cpuinit platform_boot_secondary(unsigned int cpu, struct task_struct *idle) +int platform_boot_secondary(unsigned int cpu, struct task_struct *idle) { unsigned long timeout; @@ -155,7 +155,7 @@ void platform_clear_ipi(unsigned int cpu, int irq) * Setup core B's local core timer. * In SMP, core timer is used for clock event device. */ -void __cpuinit bfin_local_timer_setup(void) +void bfin_local_timer_setup(void) { #if defined(CONFIG_TICKSOURCE_CORETMR) struct irq_data *data = irq_get_irq_data(IRQ_CORETMR); diff --git a/arch/blackfin/mach-common/cache-c.c b/arch/blackfin/mach-common/cache-c.c index a60a24f5035d3..0e1e451fd7d81 100644 --- a/arch/blackfin/mach-common/cache-c.c +++ b/arch/blackfin/mach-common/cache-c.c @@ -52,7 +52,7 @@ bfin_cache_init(struct cplb_entry *cplb_tbl, unsigned long cplb_addr, } #ifdef CONFIG_BFIN_ICACHE -void __cpuinit bfin_icache_init(struct cplb_entry *icplb_tbl) +void bfin_icache_init(struct cplb_entry *icplb_tbl) { bfin_cache_init(icplb_tbl, ICPLB_ADDR0, ICPLB_DATA0, IMEM_CONTROL, (IMC | ENICPLB)); @@ -60,7 +60,7 @@ void __cpuinit bfin_icache_init(struct cplb_entry *icplb_tbl) #endif #ifdef CONFIG_BFIN_DCACHE -void __cpuinit bfin_dcache_init(struct cplb_entry *dcplb_tbl) +void bfin_dcache_init(struct cplb_entry *dcplb_tbl) { /* * Anomaly notes: diff --git a/arch/blackfin/mach-common/ints-priority.c b/arch/blackfin/mach-common/ints-priority.c index 6c0c6816a51a1..d143fd8d2bc50 100644 --- a/arch/blackfin/mach-common/ints-priority.c +++ b/arch/blackfin/mach-common/ints-priority.c @@ -1281,7 +1281,7 @@ static struct irq_chip bfin_gpio_irqchip = { .irq_set_wake = bfin_gpio_set_wake, }; -void __cpuinit init_exception_vectors(void) +void init_exception_vectors(void) { /* cannot program in software: * evt0 - emulation (jtag) diff --git a/arch/blackfin/mach-common/smp.c b/arch/blackfin/mach-common/smp.c index 1bc2ce6f3c947..55d4cfb40faaa 100644 --- a/arch/blackfin/mach-common/smp.c +++ b/arch/blackfin/mach-common/smp.c @@ -46,7 +46,7 @@ struct corelock_slot corelock __attribute__ ((__section__(".l2.bss"))); unsigned long blackfin_iflush_l1_entry[NR_CPUS]; #endif -struct blackfin_initial_pda __cpuinitdata initial_pda_coreb; +struct blackfin_initial_pda initial_pda_coreb; enum ipi_message_type { BFIN_IPI_TIMER, @@ -249,7 +249,7 @@ void smp_send_stop(void) return; } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) +int __cpu_up(unsigned int cpu, struct task_struct *idle) { int ret; @@ -262,7 +262,7 @@ int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) return ret; } -static void __cpuinit setup_secondary(unsigned int cpu) +static void setup_secondary(unsigned int cpu) { unsigned long ilat; @@ -280,7 +280,7 @@ static void __cpuinit setup_secondary(unsigned int cpu) IMASK_IVG10 | IMASK_IVG9 | IMASK_IVG8 | IMASK_IVG7 | IMASK_IVGHW; } -void __cpuinit secondary_start_kernel(void) +void secondary_start_kernel(void) { unsigned int cpu = smp_processor_id(); struct mm_struct *mm = &init_mm; @@ -404,7 +404,7 @@ EXPORT_SYMBOL(resync_core_dcache); #endif #ifdef CONFIG_HOTPLUG_CPU -int __cpuexit __cpu_disable(void) +int __cpu_disable(void) { unsigned int cpu = smp_processor_id(); @@ -417,7 +417,7 @@ int __cpuexit __cpu_disable(void) static DECLARE_COMPLETION(cpu_killed); -int __cpuexit __cpu_die(unsigned int cpu) +int __cpu_die(unsigned int cpu) { return wait_for_completion_timeout(&cpu_killed, 5000); } diff --git a/arch/cris/arch-v32/kernel/smp.c b/arch/cris/arch-v32/kernel/smp.c index cdd12028de0c5..fe8e6039db2a0 100644 --- a/arch/cris/arch-v32/kernel/smp.c +++ b/arch/cris/arch-v32/kernel/smp.c @@ -197,7 +197,7 @@ int setup_profiling_timer(unsigned int multiplier) */ unsigned long cache_decay_ticks = 1; -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { smp_boot_one_cpu(cpu, tidle); return cpu_online(cpu) ? 0 : -ENOSYS; diff --git a/arch/frv/kernel/setup.c b/arch/frv/kernel/setup.c index a5136474c6fd6..f908aab9f3b91 100644 --- a/arch/frv/kernel/setup.c +++ b/arch/frv/kernel/setup.c @@ -709,7 +709,7 @@ static void __init reserve_dma_coherent(void) /* * calibrate the delay loop */ -void __cpuinit calibrate_delay(void) +void calibrate_delay(void) { loops_per_jiffy = __delay_loops_MHz * (1000000 / HZ); diff --git a/arch/hexagon/kernel/setup.c b/arch/hexagon/kernel/setup.c index bfe13311d70d5..29d1f1b000165 100644 --- a/arch/hexagon/kernel/setup.c +++ b/arch/hexagon/kernel/setup.c @@ -41,7 +41,7 @@ static char default_command_line[COMMAND_LINE_SIZE] __initdata = CONFIG_CMDLINE; int on_simulator; -void __cpuinit calibrate_delay(void) +void calibrate_delay(void) { loops_per_jiffy = thread_freq_mhz * 1000000 / HZ; } diff --git a/arch/hexagon/kernel/smp.c b/arch/hexagon/kernel/smp.c index 0e364ca431981..9faaa940452b9 100644 --- a/arch/hexagon/kernel/smp.c +++ b/arch/hexagon/kernel/smp.c @@ -146,7 +146,7 @@ void __init smp_prepare_boot_cpu(void) * to point to current thread info */ -void __cpuinit start_secondary(void) +void start_secondary(void) { unsigned int cpu; unsigned long thread_ptr; @@ -194,7 +194,7 @@ void __cpuinit start_secondary(void) * maintains control until "cpu_online(cpu)" is set. */ -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) +int __cpu_up(unsigned int cpu, struct task_struct *idle) { struct thread_info *thread = (struct thread_info *)idle->stack; void *stack_start; diff --git a/arch/ia64/kernel/acpi.c b/arch/ia64/kernel/acpi.c index 335eb07480fe9..5eb71d22c3d59 100644 --- a/arch/ia64/kernel/acpi.c +++ b/arch/ia64/kernel/acpi.c @@ -807,7 +807,7 @@ int acpi_isa_irq_to_gsi(unsigned isa_irq, u32 *gsi) * ACPI based hotplug CPU support */ #ifdef CONFIG_ACPI_HOTPLUG_CPU -static __cpuinit +static int acpi_map_cpu2node(acpi_handle handle, int cpu, int physid) { #ifdef CONFIG_ACPI_NUMA @@ -882,7 +882,7 @@ __init void prefill_possible_map(void) set_cpu_possible(i, true); } -static int __cpuinit _acpi_map_lsapic(acpi_handle handle, int *pcpu) +static int _acpi_map_lsapic(acpi_handle handle, int *pcpu) { struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; union acpi_object *obj; diff --git a/arch/ia64/kernel/err_inject.c b/arch/ia64/kernel/err_inject.c index 2d67317a1ec20..4dc334c71dbd6 100644 --- a/arch/ia64/kernel/err_inject.c +++ b/arch/ia64/kernel/err_inject.c @@ -225,17 +225,17 @@ static struct attribute_group err_inject_attr_group = { .name = "err_inject" }; /* Add/Remove err_inject interface for CPU device */ -static int __cpuinit err_inject_add_dev(struct device * sys_dev) +static int err_inject_add_dev(struct device * sys_dev) { return sysfs_create_group(&sys_dev->kobj, &err_inject_attr_group); } -static int __cpuinit err_inject_remove_dev(struct device * sys_dev) +static int err_inject_remove_dev(struct device * sys_dev) { sysfs_remove_group(&sys_dev->kobj, &err_inject_attr_group); return 0; } -static int __cpuinit err_inject_cpu_callback(struct notifier_block *nfb, +static int err_inject_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; @@ -256,7 +256,7 @@ static int __cpuinit err_inject_cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata err_inject_cpu_notifier = +static struct notifier_block err_inject_cpu_notifier = { .notifier_call = err_inject_cpu_callback, }; diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c index d7396dbb07bb4..b8edfa75a83f9 100644 --- a/arch/ia64/kernel/mca.c +++ b/arch/ia64/kernel/mca.c @@ -631,7 +631,7 @@ ia64_mca_register_cpev (int cpev) * Outputs * None */ -void __cpuinit +void ia64_mca_cmc_vector_setup (void) { cmcv_reg_t cmcv; @@ -1814,7 +1814,7 @@ static struct irqaction mca_cpep_irqaction = { * format most of the fields. */ -static void __cpuinit +static void format_mca_init_stack(void *mca_data, unsigned long offset, const char *type, int cpu) { @@ -1844,7 +1844,7 @@ static void * __init_refok mca_bootmem(void) } /* Do per-CPU MCA-related initialization. */ -void __cpuinit +void ia64_mca_cpu_init(void *cpu_data) { void *pal_vaddr; @@ -1896,7 +1896,7 @@ ia64_mca_cpu_init(void *cpu_data) PAGE_KERNEL)); } -static void __cpuinit ia64_mca_cmc_vector_adjust(void *dummy) +static void ia64_mca_cmc_vector_adjust(void *dummy) { unsigned long flags; @@ -1906,7 +1906,7 @@ static void __cpuinit ia64_mca_cmc_vector_adjust(void *dummy) local_irq_restore(flags); } -static int __cpuinit mca_cpu_callback(struct notifier_block *nfb, +static int mca_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { @@ -1922,7 +1922,7 @@ static int __cpuinit mca_cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block mca_cpu_notifier __cpuinitdata = { +static struct notifier_block mca_cpu_notifier = { .notifier_call = mca_cpu_callback }; diff --git a/arch/ia64/kernel/numa.c b/arch/ia64/kernel/numa.c index c93420c97409d..d288cde936066 100644 --- a/arch/ia64/kernel/numa.c +++ b/arch/ia64/kernel/numa.c @@ -30,7 +30,7 @@ EXPORT_SYMBOL(cpu_to_node_map); cpumask_t node_to_cpu_mask[MAX_NUMNODES] __cacheline_aligned; EXPORT_SYMBOL(node_to_cpu_mask); -void __cpuinit map_cpu_to_node(int cpu, int nid) +void map_cpu_to_node(int cpu, int nid) { int oldnid; if (nid < 0) { /* just initialize by zero */ @@ -51,7 +51,7 @@ void __cpuinit map_cpu_to_node(int cpu, int nid) return; } -void __cpuinit unmap_cpu_from_node(int cpu, int nid) +void unmap_cpu_from_node(int cpu, int nid) { WARN_ON(!cpu_isset(cpu, node_to_cpu_mask[nid])); WARN_ON(cpu_to_node_map[cpu] != nid); diff --git a/arch/ia64/kernel/palinfo.c b/arch/ia64/kernel/palinfo.c index 2b3c2d79256f9..ab333284f4b2e 100644 --- a/arch/ia64/kernel/palinfo.c +++ b/arch/ia64/kernel/palinfo.c @@ -932,7 +932,7 @@ static const struct file_operations proc_palinfo_fops = { .release = single_release, }; -static void __cpuinit +static void create_palinfo_proc_entries(unsigned int cpu) { pal_func_cpu_u_t f; @@ -962,7 +962,7 @@ remove_palinfo_proc_entries(unsigned int hcpu) remove_proc_subtree(cpustr, palinfo_dir); } -static int __cpuinit palinfo_cpu_callback(struct notifier_block *nfb, +static int palinfo_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int hotcpu = (unsigned long)hcpu; diff --git a/arch/ia64/kernel/salinfo.c b/arch/ia64/kernel/salinfo.c index 4bc580af67b39..960a396f59296 100644 --- a/arch/ia64/kernel/salinfo.c +++ b/arch/ia64/kernel/salinfo.c @@ -568,7 +568,7 @@ static const struct file_operations salinfo_data_fops = { .llseek = default_llseek, }; -static int __cpuinit +static int salinfo_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu) { unsigned int i, cpu = (unsigned long)hcpu; @@ -609,7 +609,7 @@ salinfo_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu return NOTIFY_OK; } -static struct notifier_block salinfo_cpu_notifier __cpuinitdata = +static struct notifier_block salinfo_cpu_notifier = { .notifier_call = salinfo_cpu_callback, .priority = 0, diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c index 13bfdd22afc8e..fff7d768c669f 100644 --- a/arch/ia64/kernel/setup.c +++ b/arch/ia64/kernel/setup.c @@ -748,7 +748,7 @@ const struct seq_operations cpuinfo_op = { #define MAX_BRANDS 8 static char brandname[MAX_BRANDS][128]; -static char * __cpuinit +static char * get_model_name(__u8 family, __u8 model) { static int overflow; @@ -778,7 +778,7 @@ get_model_name(__u8 family, __u8 model) return "Unknown"; } -static void __cpuinit +static void identify_cpu (struct cpuinfo_ia64 *c) { union { @@ -850,7 +850,7 @@ identify_cpu (struct cpuinfo_ia64 *c) * 2. the minimum of the i-cache stride sizes for "flush_icache_range()". * 3. the minimum of the cache stride sizes for "clflush_cache_range()". */ -static void __cpuinit +static void get_cache_info(void) { unsigned long line_size, max = 1; @@ -915,10 +915,10 @@ get_cache_info(void) * cpu_init() initializes state that is per-CPU. This function acts * as a 'CPU state barrier', nothing should get across. */ -void __cpuinit +void cpu_init (void) { - extern void __cpuinit ia64_mmu_init (void *); + extern void ia64_mmu_init (void *); static unsigned long max_num_phys_stacked = IA64_NUM_PHYS_STACK_REG; unsigned long num_phys_stacked; pal_vm_info_2_u_t vmi; diff --git a/arch/ia64/kernel/smpboot.c b/arch/ia64/kernel/smpboot.c index 8d87168d218df..547a48d78bd7f 100644 --- a/arch/ia64/kernel/smpboot.c +++ b/arch/ia64/kernel/smpboot.c @@ -351,7 +351,7 @@ static inline void smp_setup_percpu_timer(void) { } -static void __cpuinit +static void smp_callin (void) { int cpuid, phys_id, itc_master; @@ -442,7 +442,7 @@ smp_callin (void) /* * Activate a secondary processor. head.S calls this. */ -int __cpuinit +int start_secondary (void *unused) { /* Early console may use I/O ports */ @@ -459,7 +459,7 @@ start_secondary (void *unused) return 0; } -static int __cpuinit +static int do_boot_cpu (int sapicid, int cpu, struct task_struct *idle) { int timeout; @@ -728,7 +728,7 @@ static inline void set_cpu_sibling_map(int cpu) } } -int __cpuinit +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { int ret; diff --git a/arch/ia64/kernel/topology.c b/arch/ia64/kernel/topology.c index dc00b2c1b42ab..33980354ee466 100644 --- a/arch/ia64/kernel/topology.c +++ b/arch/ia64/kernel/topology.c @@ -139,7 +139,7 @@ static struct cpu_cache_info all_cpu_cache_info[NR_CPUS] __cpuinitdata; #define LEAF_KOBJECT_PTR(x,y) (&all_cpu_cache_info[x].cache_leaves[y]) #ifdef CONFIG_SMP -static void __cpuinit cache_shared_cpu_map_setup( unsigned int cpu, +static void cache_shared_cpu_map_setup( unsigned int cpu, struct cache_info * this_leaf) { pal_cache_shared_info_t csi; @@ -174,7 +174,7 @@ static void __cpuinit cache_shared_cpu_map_setup( unsigned int cpu, &csi) == PAL_STATUS_SUCCESS); } #else -static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, +static void cache_shared_cpu_map_setup(unsigned int cpu, struct cache_info * this_leaf) { cpu_set(cpu, this_leaf->shared_cpu_map); @@ -298,7 +298,7 @@ static struct kobj_type cache_ktype_percpu_entry = { .sysfs_ops = &cache_sysfs_ops, }; -static void __cpuinit cpu_cache_sysfs_exit(unsigned int cpu) +static void cpu_cache_sysfs_exit(unsigned int cpu) { kfree(all_cpu_cache_info[cpu].cache_leaves); all_cpu_cache_info[cpu].cache_leaves = NULL; @@ -307,7 +307,7 @@ static void __cpuinit cpu_cache_sysfs_exit(unsigned int cpu) return; } -static int __cpuinit cpu_cache_sysfs_init(unsigned int cpu) +static int cpu_cache_sysfs_init(unsigned int cpu) { unsigned long i, levels, unique_caches; pal_cache_config_info_t cci; @@ -351,7 +351,7 @@ static int __cpuinit cpu_cache_sysfs_init(unsigned int cpu) } /* Add cache interface for CPU device */ -static int __cpuinit cache_add_dev(struct device * sys_dev) +static int cache_add_dev(struct device * sys_dev) { unsigned int cpu = sys_dev->id; unsigned long i, j; @@ -401,7 +401,7 @@ static int __cpuinit cache_add_dev(struct device * sys_dev) } /* Remove cache interface for CPU device */ -static int __cpuinit cache_remove_dev(struct device * sys_dev) +static int cache_remove_dev(struct device * sys_dev) { unsigned int cpu = sys_dev->id; unsigned long i; @@ -425,7 +425,7 @@ static int __cpuinit cache_remove_dev(struct device * sys_dev) * When a cpu is hot-plugged, do a check and initiate * cache kobject if necessary */ -static int __cpuinit cache_cpu_callback(struct notifier_block *nfb, +static int cache_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; @@ -445,7 +445,7 @@ static int __cpuinit cache_cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata cache_cpu_notifier = +static struct notifier_block cache_cpu_notifier = { .notifier_call = cache_cpu_callback }; diff --git a/arch/ia64/mm/contig.c b/arch/ia64/mm/contig.c index 67c59ebec899c..41d83fe5ee4de 100644 --- a/arch/ia64/mm/contig.c +++ b/arch/ia64/mm/contig.c @@ -156,7 +156,7 @@ static void *cpu_data; * * Allocate and setup per-cpu data areas. */ -void * __cpuinit +void * per_cpu_init (void) { static bool first_time = true; diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c index ae4db4bd6d97d..7253d83650bf8 100644 --- a/arch/ia64/mm/discontig.c +++ b/arch/ia64/mm/discontig.c @@ -593,7 +593,7 @@ void __init find_memory(void) * find_pernode_space() does most of this already, we just need to set * local_per_cpu_offset */ -void __cpuinit *per_cpu_init(void) +void *per_cpu_init(void) { int cpu; static int first_time = 1; diff --git a/arch/ia64/mm/numa.c b/arch/ia64/mm/numa.c index 4248492b9321b..ea21d4cad540e 100644 --- a/arch/ia64/mm/numa.c +++ b/arch/ia64/mm/numa.c @@ -86,7 +86,7 @@ int __meminit __early_pfn_to_nid(unsigned long pfn) return -1; } -void __cpuinit numa_clear_node(int cpu) +void numa_clear_node(int cpu) { unmap_cpu_from_node(cpu, NUMA_NO_NODE); } diff --git a/arch/ia64/sn/kernel/setup.c b/arch/ia64/sn/kernel/setup.c index f82e7b462b7b1..53b01b8e2f197 100644 --- a/arch/ia64/sn/kernel/setup.c +++ b/arch/ia64/sn/kernel/setup.c @@ -192,7 +192,7 @@ void __init early_sn_setup(void) } extern int platform_intr_list[]; -static int __cpuinitdata shub_1_1_found; +static int shub_1_1_found; /* * sn_check_for_wars @@ -200,7 +200,7 @@ static int __cpuinitdata shub_1_1_found; * Set flag for enabling shub specific wars */ -static inline int __cpuinit is_shub_1_1(int nasid) +static inline int is_shub_1_1(int nasid) { unsigned long id; int rev; @@ -212,7 +212,7 @@ static inline int __cpuinit is_shub_1_1(int nasid) return rev <= 2; } -static void __cpuinit sn_check_for_wars(void) +static void sn_check_for_wars(void) { int cnode; @@ -558,7 +558,7 @@ static void __init sn_init_pdas(char **cmdline_p) * Also sets up a few fields in the nodepda. Also known as * platform_cpu_init() by the ia64 machvec code. */ -void __cpuinit sn_cpu_init(void) +void sn_cpu_init(void) { int cpuid; int cpuphyid; diff --git a/arch/ia64/xen/hypervisor.c b/arch/ia64/xen/hypervisor.c index 52172eee85913..fab62528a80b4 100644 --- a/arch/ia64/xen/hypervisor.c +++ b/arch/ia64/xen/hypervisor.c @@ -74,7 +74,7 @@ void __init xen_setup_vcpu_info_placement(void) xen_vcpu_setup(cpu); } -void __cpuinit +void xen_cpu_init(void) { xen_smp_intr_init(); diff --git a/arch/m32r/kernel/smpboot.c b/arch/m32r/kernel/smpboot.c index 0ac558adc6055..bb21f4f631700 100644 --- a/arch/m32r/kernel/smpboot.c +++ b/arch/m32r/kernel/smpboot.c @@ -343,7 +343,7 @@ static void __init do_boot_cpu(int phys_id) } } -int __cpuinit __cpu_up(unsigned int cpu_id, struct task_struct *tidle) +int __cpu_up(unsigned int cpu_id, struct task_struct *tidle) { int timeout; diff --git a/arch/metag/kernel/perf/perf_event.c b/arch/metag/kernel/perf/perf_event.c index 366569425c52a..49ae6ff4bac69 100644 --- a/arch/metag/kernel/perf/perf_event.c +++ b/arch/metag/kernel/perf/perf_event.c @@ -813,7 +813,7 @@ static struct metag_pmu _metag_pmu = { }; /* PMU CPU hotplug notifier */ -static int __cpuinit metag_pmu_cpu_notify(struct notifier_block *b, +static int metag_pmu_cpu_notify(struct notifier_block *b, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned int)hcpu; @@ -828,7 +828,7 @@ static int __cpuinit metag_pmu_cpu_notify(struct notifier_block *b, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata metag_pmu_notifier = { +static struct notifier_block metag_pmu_notifier = { .notifier_call = metag_pmu_cpu_notify, }; diff --git a/arch/metag/kernel/smp.c b/arch/metag/kernel/smp.c index f443ec9a7cbef..6f9d5ac915c76 100644 --- a/arch/metag/kernel/smp.c +++ b/arch/metag/kernel/smp.c @@ -65,7 +65,7 @@ static DEFINE_SPINLOCK(boot_lock); /* * "thread" is assumed to be a valid Meta hardware thread ID. */ -int __cpuinit boot_secondary(unsigned int thread, struct task_struct *idle) +int boot_secondary(unsigned int thread, struct task_struct *idle) { u32 val; @@ -115,7 +115,7 @@ int __cpuinit boot_secondary(unsigned int thread, struct task_struct *idle) * If the cache partition has changed, prints a message to the log describing * those changes. */ -static __cpuinit void describe_cachepart_change(unsigned int thread, +static void describe_cachepart_change(unsigned int thread, const char *label, unsigned int sz, unsigned int old, @@ -167,7 +167,7 @@ static __cpuinit void describe_cachepart_change(unsigned int thread, * Ensures that coherency is enabled and that the threads share the same cache * partitions. */ -static __cpuinit void setup_smp_cache(unsigned int thread) +static void setup_smp_cache(unsigned int thread) { unsigned int this_thread, lflags; unsigned int dcsz, dcpart_this, dcpart_old, dcpart_new; @@ -212,7 +212,7 @@ static __cpuinit void setup_smp_cache(unsigned int thread) icpart_old, icpart_new); } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) +int __cpu_up(unsigned int cpu, struct task_struct *idle) { unsigned int thread = cpu_2_hwthread_id[cpu]; int ret; @@ -273,7 +273,7 @@ static DECLARE_COMPLETION(cpu_killed); /* * __cpu_disable runs on the processor to be shutdown. */ -int __cpuexit __cpu_disable(void) +int __cpu_disable(void) { unsigned int cpu = smp_processor_id(); struct task_struct *p; @@ -310,7 +310,7 @@ int __cpuexit __cpu_disable(void) * called on the thread which is asking for a CPU to be shutdown - * waits until shutdown has completed, or it is timed out. */ -void __cpuexit __cpu_die(unsigned int cpu) +void __cpu_die(unsigned int cpu) { if (!wait_for_completion_timeout(&cpu_killed, msecs_to_jiffies(1))) pr_err("CPU%u: unable to kill\n", cpu); @@ -322,7 +322,7 @@ void __cpuexit __cpu_die(unsigned int cpu) * Note that we do not return from this function. If this cpu is * brought online again it will need to run secondary_startup(). */ -void __cpuexit cpu_die(void) +void cpu_die(void) { local_irq_disable(); idle_task_exit(); @@ -337,7 +337,7 @@ void __cpuexit cpu_die(void) * Called by both boot and secondaries to move global data into * per-processor storage. */ -void __cpuinit smp_store_cpu_info(unsigned int cpuid) +void smp_store_cpu_info(unsigned int cpuid) { struct cpuinfo_metag *cpu_info = &per_cpu(cpu_data, cpuid); diff --git a/arch/metag/kernel/traps.c b/arch/metag/kernel/traps.c index 2ceeaae5b1992..a645da359a3f4 100644 --- a/arch/metag/kernel/traps.c +++ b/arch/metag/kernel/traps.c @@ -811,7 +811,7 @@ static void set_trigger_mask(unsigned int mask) } #endif -void __cpuinit per_cpu_trap_init(unsigned long cpu) +void per_cpu_trap_init(unsigned long cpu) { TBIRES int_context; unsigned int thread = cpu_2_hwthread_id[cpu]; diff --git a/arch/mips/ath79/setup.c b/arch/mips/ath79/setup.c index 8be4e856b8b89..80f4ecd42b0dc 100644 --- a/arch/mips/ath79/setup.c +++ b/arch/mips/ath79/setup.c @@ -182,7 +182,7 @@ const char *get_system_type(void) return ath79_sys_type; } -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { return CP0_LEGACY_COMPARE_IRQ; } diff --git a/arch/mips/cavium-octeon/octeon-irq.c b/arch/mips/cavium-octeon/octeon-irq.c index 45c1a6caa2066..5e3da374caab1 100644 --- a/arch/mips/cavium-octeon/octeon-irq.c +++ b/arch/mips/cavium-octeon/octeon-irq.c @@ -1095,7 +1095,7 @@ static void octeon_irq_ip3_ciu(void) static bool octeon_irq_use_ip4; -static void __cpuinit octeon_irq_local_enable_ip4(void *arg) +static void octeon_irq_local_enable_ip4(void *arg) { set_c0_status(STATUSF_IP4); } @@ -1110,21 +1110,21 @@ static void (*octeon_irq_ip2)(void); static void (*octeon_irq_ip3)(void); static void (*octeon_irq_ip4)(void); -void __cpuinitdata (*octeon_irq_setup_secondary)(void); +void (*octeon_irq_setup_secondary)(void); -void __cpuinit octeon_irq_set_ip4_handler(octeon_irq_ip4_handler_t h) +void octeon_irq_set_ip4_handler(octeon_irq_ip4_handler_t h) { octeon_irq_ip4 = h; octeon_irq_use_ip4 = true; on_each_cpu(octeon_irq_local_enable_ip4, NULL, 1); } -static void __cpuinit octeon_irq_percpu_enable(void) +static void octeon_irq_percpu_enable(void) { irq_cpu_online(); } -static void __cpuinit octeon_irq_init_ciu_percpu(void) +static void octeon_irq_init_ciu_percpu(void) { int coreid = cvmx_get_core_num(); @@ -1167,7 +1167,7 @@ static void octeon_irq_init_ciu2_percpu(void) cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(coreid)); } -static void __cpuinit octeon_irq_setup_secondary_ciu(void) +static void octeon_irq_setup_secondary_ciu(void) { octeon_irq_init_ciu_percpu(); octeon_irq_percpu_enable(); diff --git a/arch/mips/cavium-octeon/smp.c b/arch/mips/cavium-octeon/smp.c index 295137dfdc37c..138cc80c59281 100644 --- a/arch/mips/cavium-octeon/smp.c +++ b/arch/mips/cavium-octeon/smp.c @@ -173,7 +173,7 @@ static void octeon_boot_secondary(int cpu, struct task_struct *idle) * After we've done initial boot, this function is called to allow the * board code to clean up state, if needed */ -static void __cpuinit octeon_init_secondary(void) +static void octeon_init_secondary(void) { unsigned int sr; @@ -375,7 +375,7 @@ static int octeon_update_boot_vector(unsigned int cpu) return 0; } -static int __cpuinit octeon_cpu_callback(struct notifier_block *nfb, +static int octeon_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; @@ -394,7 +394,7 @@ static int __cpuinit octeon_cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static int __cpuinit register_cavium_notifier(void) +static int register_cavium_notifier(void) { hotcpu_notifier(octeon_cpu_callback, 0); return 0; diff --git a/arch/mips/include/asm/uasm.h b/arch/mips/include/asm/uasm.h index 370d967725c28..dc4436e7132f5 100644 --- a/arch/mips/include/asm/uasm.h +++ b/arch/mips/include/asm/uasm.h @@ -17,8 +17,8 @@ #define __uasminitdata #define UASM_EXPORT_SYMBOL(sym) EXPORT_SYMBOL(sym) #else -#define __uasminit __cpuinit -#define __uasminitdata __cpuinitdata +#define __uasminit +#define __uasminitdatadata #define UASM_EXPORT_SYMBOL(sym) #endif diff --git a/arch/mips/kernel/cevt-bcm1480.c b/arch/mips/kernel/cevt-bcm1480.c index 15f618b40cf6c..7976457184b18 100644 --- a/arch/mips/kernel/cevt-bcm1480.c +++ b/arch/mips/kernel/cevt-bcm1480.c @@ -109,7 +109,7 @@ static DEFINE_PER_CPU(struct clock_event_device, sibyte_hpt_clockevent); static DEFINE_PER_CPU(struct irqaction, sibyte_hpt_irqaction); static DEFINE_PER_CPU(char [18], sibyte_hpt_name); -void __cpuinit sb1480_clockevent_init(void) +void sb1480_clockevent_init(void) { unsigned int cpu = smp_processor_id(); unsigned int irq = K_BCM1480_INT_TIMER_0 + cpu; diff --git a/arch/mips/kernel/cevt-gic.c b/arch/mips/kernel/cevt-gic.c index 730eaf92c0189..594cbbf16d627 100644 --- a/arch/mips/kernel/cevt-gic.c +++ b/arch/mips/kernel/cevt-gic.c @@ -59,7 +59,7 @@ void gic_event_handler(struct clock_event_device *dev) { } -int __cpuinit gic_clockevent_init(void) +int gic_clockevent_init(void) { unsigned int cpu = smp_processor_id(); struct clock_event_device *cd; diff --git a/arch/mips/kernel/cevt-r4k.c b/arch/mips/kernel/cevt-r4k.c index 02033eaf88254..50d3f5a8d6bb1 100644 --- a/arch/mips/kernel/cevt-r4k.c +++ b/arch/mips/kernel/cevt-r4k.c @@ -171,7 +171,7 @@ int c0_compare_int_usable(void) } #ifndef CONFIG_MIPS_MT_SMTC -int __cpuinit r4k_clockevent_init(void) +int r4k_clockevent_init(void) { unsigned int cpu = smp_processor_id(); struct clock_event_device *cd; diff --git a/arch/mips/kernel/cevt-sb1250.c b/arch/mips/kernel/cevt-sb1250.c index 200f2778bf366..5ea6d6b1de150 100644 --- a/arch/mips/kernel/cevt-sb1250.c +++ b/arch/mips/kernel/cevt-sb1250.c @@ -107,7 +107,7 @@ static DEFINE_PER_CPU(struct clock_event_device, sibyte_hpt_clockevent); static DEFINE_PER_CPU(struct irqaction, sibyte_hpt_irqaction); static DEFINE_PER_CPU(char [18], sibyte_hpt_name); -void __cpuinit sb1250_clockevent_init(void) +void sb1250_clockevent_init(void) { unsigned int cpu = smp_processor_id(); unsigned int irq = K_INT_TIMER_0 + cpu; diff --git a/arch/mips/kernel/cevt-smtc.c b/arch/mips/kernel/cevt-smtc.c index 9de5ed7ef1a36..b6cf0a60d8968 100644 --- a/arch/mips/kernel/cevt-smtc.c +++ b/arch/mips/kernel/cevt-smtc.c @@ -248,7 +248,7 @@ irqreturn_t c0_compare_interrupt(int irq, void *dev_id) } -int __cpuinit smtc_clockevent_init(void) +int smtc_clockevent_init(void) { uint64_t mips_freq = mips_hpt_frequency; unsigned int cpu = smp_processor_id(); diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index c6568bf4b1b05..d866a45b551f2 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -27,7 +27,7 @@ #include #include -static int __cpuinitdata mips_fpu_disabled; +static int mips_fpu_disabled; static int __init fpu_disable(char *s) { @@ -39,7 +39,7 @@ static int __init fpu_disable(char *s) __setup("nofpu", fpu_disable); -int __cpuinitdata mips_dsp_disabled; +int mips_dsp_disabled; static int __init dsp_disable(char *s) { @@ -134,7 +134,7 @@ static inline void cpu_probe_vmbits(struct cpuinfo_mips *c) #endif } -static void __cpuinit set_isa(struct cpuinfo_mips *c, unsigned int isa) +static void set_isa(struct cpuinfo_mips *c, unsigned int isa) { switch (isa) { case MIPS_CPU_ISA_M64R2: @@ -162,7 +162,7 @@ static void __cpuinit set_isa(struct cpuinfo_mips *c, unsigned int isa) } } -static char unknown_isa[] __cpuinitdata = KERN_ERR \ +static char unknown_isa[] = KERN_ERR \ "Unsupported ISA type, c0.config0: %d."; static inline unsigned int decode_config0(struct cpuinfo_mips *c) @@ -296,7 +296,7 @@ static inline unsigned int decode_config4(struct cpuinfo_mips *c) return config4 & MIPS_CONF_M; } -static void __cpuinit decode_configs(struct cpuinfo_mips *c) +static void decode_configs(struct cpuinfo_mips *c) { int ok; @@ -970,7 +970,7 @@ EXPORT_SYMBOL(__ua_limit); const char *__cpu_name[NR_CPUS]; const char *__elf_platform; -__cpuinit void cpu_probe(void) +void cpu_probe(void) { struct cpuinfo_mips *c = ¤t_cpu_data; unsigned int cpu = smp_processor_id(); @@ -1055,7 +1055,7 @@ __cpuinit void cpu_probe(void) #endif } -__cpuinit void cpu_report(void) +void cpu_report(void) { struct cpuinfo_mips *c = ¤t_cpu_data; diff --git a/arch/mips/kernel/smp-bmips.c b/arch/mips/kernel/smp-bmips.c index 8e393b8443f7a..20c637fdae892 100644 --- a/arch/mips/kernel/smp-bmips.c +++ b/arch/mips/kernel/smp-bmips.c @@ -381,7 +381,7 @@ struct plat_smp_ops bmips_smp_ops = { * UP BMIPS systems as well. ***********************************************************************/ -static void __cpuinit bmips_wr_vec(unsigned long dst, char *start, char *end) +static void bmips_wr_vec(unsigned long dst, char *start, char *end) { memcpy((void *)dst, start, end - start); dma_cache_wback((unsigned long)start, end - start); @@ -389,7 +389,7 @@ static void __cpuinit bmips_wr_vec(unsigned long dst, char *start, char *end) instruction_hazard(); } -static inline void __cpuinit bmips_nmi_handler_setup(void) +static inline void bmips_nmi_handler_setup(void) { bmips_wr_vec(BMIPS_NMI_RESET_VEC, &bmips_reset_nmi_vec, &bmips_reset_nmi_vec_end); @@ -397,7 +397,7 @@ static inline void __cpuinit bmips_nmi_handler_setup(void) &bmips_smp_int_vec_end); } -void __cpuinit bmips_ebase_setup(void) +void bmips_ebase_setup(void) { unsigned long new_ebase = ebase; void __iomem __maybe_unused *cbr; diff --git a/arch/mips/kernel/smp-mt.c b/arch/mips/kernel/smp-mt.c index 3e5164c11caca..57a3f7a2b370c 100644 --- a/arch/mips/kernel/smp-mt.c +++ b/arch/mips/kernel/smp-mt.c @@ -149,7 +149,7 @@ static void vsmp_send_ipi_mask(const struct cpumask *mask, unsigned int action) vsmp_send_ipi_single(i, action); } -static void __cpuinit vsmp_init_secondary(void) +static void vsmp_init_secondary(void) { #ifdef CONFIG_IRQ_GIC /* This is Malta specific: IPI,performance and timer interrupts */ @@ -162,7 +162,7 @@ static void __cpuinit vsmp_init_secondary(void) STATUSF_IP6 | STATUSF_IP7); } -static void __cpuinit vsmp_smp_finish(void) +static void vsmp_smp_finish(void) { /* CDFIXME: remove this? */ write_c0_compare(read_c0_count() + (8* mips_hpt_frequency/HZ)); @@ -188,7 +188,7 @@ static void vsmp_cpus_done(void) * (unsigned long)idle->thread_info the gp * assumes a 1:1 mapping of TC => VPE */ -static void __cpuinit vsmp_boot_secondary(int cpu, struct task_struct *idle) +static void vsmp_boot_secondary(int cpu, struct task_struct *idle) { struct thread_info *gp = task_thread_info(idle); dvpe(); diff --git a/arch/mips/kernel/smp-up.c b/arch/mips/kernel/smp-up.c index 00500fea27507..7fde3e4d978f9 100644 --- a/arch/mips/kernel/smp-up.c +++ b/arch/mips/kernel/smp-up.c @@ -28,11 +28,11 @@ static inline void up_send_ipi_mask(const struct cpumask *mask, * After we've done initial boot, this function is called to allow the * board code to clean up state, if needed */ -static void __cpuinit up_init_secondary(void) +static void up_init_secondary(void) { } -static void __cpuinit up_smp_finish(void) +static void up_smp_finish(void) { } @@ -44,7 +44,7 @@ static void up_cpus_done(void) /* * Firmware CPU startup hook */ -static void __cpuinit up_boot_secondary(int cpu, struct task_struct *idle) +static void up_boot_secondary(int cpu, struct task_struct *idle) { } diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c index caeec21df1c4b..57b89cba16245 100644 --- a/arch/mips/kernel/smp.c +++ b/arch/mips/kernel/smp.c @@ -86,7 +86,7 @@ static inline void set_cpu_sibling_map(int cpu) struct plat_smp_ops *mp_ops; EXPORT_SYMBOL(mp_ops); -__cpuinit void register_smp_ops(struct plat_smp_ops *ops) +void register_smp_ops(struct plat_smp_ops *ops) { if (mp_ops) printk(KERN_WARNING "Overriding previously set SMP ops\n"); @@ -98,7 +98,7 @@ __cpuinit void register_smp_ops(struct plat_smp_ops *ops) * First C code run on the secondary CPUs after being started up by * the master. */ -asmlinkage __cpuinit void start_secondary(void) +asmlinkage void start_secondary(void) { unsigned int cpu; @@ -197,7 +197,7 @@ void smp_prepare_boot_cpu(void) cpu_set(0, cpu_callin_map); } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { mp_ops->boot_secondary(cpu, tidle); diff --git a/arch/mips/kernel/smtc.c b/arch/mips/kernel/smtc.c index 75a4fd709841a..dfc1b911be04b 100644 --- a/arch/mips/kernel/smtc.c +++ b/arch/mips/kernel/smtc.c @@ -645,7 +645,7 @@ void smtc_prepare_cpus(int cpus) * (unsigned long)idle->thread_info the gp * */ -void __cpuinit smtc_boot_secondary(int cpu, struct task_struct *idle) +void smtc_boot_secondary(int cpu, struct task_struct *idle) { extern u32 kernelsp[NR_CPUS]; unsigned long flags; diff --git a/arch/mips/kernel/spram.c b/arch/mips/kernel/spram.c index 6af08d896e20b..93f86817f20a6 100644 --- a/arch/mips/kernel/spram.c +++ b/arch/mips/kernel/spram.c @@ -37,7 +37,7 @@ /* * Different semantics to the set_c0_* function built by __BUILD_SET_C0 */ -static __cpuinit unsigned int bis_c0_errctl(unsigned int set) +static unsigned int bis_c0_errctl(unsigned int set) { unsigned int res; res = read_c0_errctl(); @@ -45,7 +45,7 @@ static __cpuinit unsigned int bis_c0_errctl(unsigned int set) return res; } -static __cpuinit void ispram_store_tag(unsigned int offset, unsigned int data) +static void ispram_store_tag(unsigned int offset, unsigned int data) { unsigned int errctl; @@ -64,7 +64,7 @@ static __cpuinit void ispram_store_tag(unsigned int offset, unsigned int data) } -static __cpuinit unsigned int ispram_load_tag(unsigned int offset) +static unsigned int ispram_load_tag(unsigned int offset) { unsigned int data; unsigned int errctl; @@ -82,7 +82,7 @@ static __cpuinit unsigned int ispram_load_tag(unsigned int offset) return data; } -static __cpuinit void dspram_store_tag(unsigned int offset, unsigned int data) +static void dspram_store_tag(unsigned int offset, unsigned int data) { unsigned int errctl; @@ -98,7 +98,7 @@ static __cpuinit void dspram_store_tag(unsigned int offset, unsigned int data) } -static __cpuinit unsigned int dspram_load_tag(unsigned int offset) +static unsigned int dspram_load_tag(unsigned int offset) { unsigned int data; unsigned int errctl; @@ -115,7 +115,7 @@ static __cpuinit unsigned int dspram_load_tag(unsigned int offset) return data; } -static __cpuinit void probe_spram(char *type, +static void probe_spram(char *type, unsigned int base, unsigned int (*read)(unsigned int), void (*write)(unsigned int, unsigned int)) @@ -196,7 +196,7 @@ static __cpuinit void probe_spram(char *type, offset += 2 * SPRAM_TAG_STRIDE; } } -void __cpuinit spram_config(void) +void spram_config(void) { struct cpuinfo_mips *c = ¤t_cpu_data; unsigned int config0; diff --git a/arch/mips/kernel/sync-r4k.c b/arch/mips/kernel/sync-r4k.c index 1ff43d5ac2c4e..84536bf4a1540 100644 --- a/arch/mips/kernel/sync-r4k.c +++ b/arch/mips/kernel/sync-r4k.c @@ -20,15 +20,15 @@ #include #include -static atomic_t __cpuinitdata count_start_flag = ATOMIC_INIT(0); -static atomic_t __cpuinitdata count_count_start = ATOMIC_INIT(0); -static atomic_t __cpuinitdata count_count_stop = ATOMIC_INIT(0); -static atomic_t __cpuinitdata count_reference = ATOMIC_INIT(0); +static atomic_t count_start_flag = ATOMIC_INIT(0); +static atomic_t count_count_start = ATOMIC_INIT(0); +static atomic_t count_count_stop = ATOMIC_INIT(0); +static atomic_t count_reference = ATOMIC_INIT(0); #define COUNTON 100 #define NR_LOOPS 5 -void __cpuinit synchronise_count_master(int cpu) +void synchronise_count_master(int cpu) { int i; unsigned long flags; @@ -106,7 +106,7 @@ void __cpuinit synchronise_count_master(int cpu) printk("done.\n"); } -void __cpuinit synchronise_count_slave(int cpu) +void synchronise_count_slave(int cpu) { int i; unsigned int initcount; diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index a75ae40184aa3..4846c221ceec7 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -1642,7 +1642,7 @@ int cp0_compare_irq_shift; int cp0_perfcount_irq; EXPORT_SYMBOL_GPL(cp0_perfcount_irq); -static int __cpuinitdata noulri; +static int noulri; static int __init ulri_disable(char *s) { @@ -1653,7 +1653,7 @@ static int __init ulri_disable(char *s) } __setup("noulri", ulri_disable); -void __cpuinit per_cpu_trap_init(bool is_boot_cpu) +void per_cpu_trap_init(bool is_boot_cpu) { unsigned int cpu = smp_processor_id(); unsigned int status_set = ST0_CU0; @@ -1770,7 +1770,7 @@ void __cpuinit per_cpu_trap_init(bool is_boot_cpu) } /* Install CPU exception handler */ -void __cpuinit set_handler(unsigned long offset, void *addr, unsigned long size) +void set_handler(unsigned long offset, void *addr, unsigned long size) { #ifdef CONFIG_CPU_MICROMIPS memcpy((void *)(ebase + offset), ((unsigned char *)addr - 1), size); @@ -1780,7 +1780,7 @@ void __cpuinit set_handler(unsigned long offset, void *addr, unsigned long size) local_flush_icache_range(ebase + offset, ebase + offset + size); } -static char panic_null_cerr[] __cpuinitdata = +static char panic_null_cerr[] = "Trying to set NULL cache error exception handler"; /* @@ -1788,7 +1788,7 @@ static char panic_null_cerr[] __cpuinitdata = * This is suitable only for the cache error exception which is the only * exception handler that is being run uncached. */ -void __cpuinit set_uncached_handler(unsigned long offset, void *addr, +void set_uncached_handler(unsigned long offset, void *addr, unsigned long size) { unsigned long uncached_ebase = CKSEG1ADDR(ebase); diff --git a/arch/mips/kernel/watch.c b/arch/mips/kernel/watch.c index 7726f6157d9e7..3b7b9003ae2aa 100644 --- a/arch/mips/kernel/watch.c +++ b/arch/mips/kernel/watch.c @@ -100,7 +100,7 @@ void mips_clear_watch_registers(void) } } -__cpuinit void mips_probe_watch_registers(struct cpuinfo_mips *c) +void mips_probe_watch_registers(struct cpuinfo_mips *c) { unsigned int t; diff --git a/arch/mips/lantiq/irq.c b/arch/mips/lantiq/irq.c index 51194875f1582..eb3e186596304 100644 --- a/arch/mips/lantiq/irq.c +++ b/arch/mips/lantiq/irq.c @@ -461,7 +461,7 @@ int __init icu_of_init(struct device_node *node, struct device_node *parent) return 0; } -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { return MIPS_CPU_TIMER_IRQ; } diff --git a/arch/mips/lib/uncached.c b/arch/mips/lib/uncached.c index 65e3dfc4e5855..d8522f8e842a2 100644 --- a/arch/mips/lib/uncached.c +++ b/arch/mips/lib/uncached.c @@ -36,7 +36,7 @@ * values, so we can avoid sharing the same stack area between a cached * and the uncached mode. */ -unsigned long __cpuinit run_uncached(void *func) +unsigned long run_uncached(void *func) { register long sp __asm__("$sp"); register long ret __asm__("$2"); diff --git a/arch/mips/mm/c-octeon.c b/arch/mips/mm/c-octeon.c index 8557fb5528632..a0bcdbb81d410 100644 --- a/arch/mips/mm/c-octeon.c +++ b/arch/mips/mm/c-octeon.c @@ -180,7 +180,7 @@ static void octeon_flush_kernel_vmap_range(unsigned long vaddr, int size) * Probe Octeon's caches * */ -static void __cpuinit probe_octeon(void) +static void probe_octeon(void) { unsigned long icache_size; unsigned long dcache_size; @@ -251,7 +251,7 @@ static void __cpuinit probe_octeon(void) } } -static void __cpuinit octeon_cache_error_setup(void) +static void octeon_cache_error_setup(void) { extern char except_vec2_octeon; set_handler(0x100, &except_vec2_octeon, 0x80); @@ -261,7 +261,7 @@ static void __cpuinit octeon_cache_error_setup(void) * Setup the Octeon cache flush routines * */ -void __cpuinit octeon_cache_init(void) +void octeon_cache_init(void) { probe_octeon(); diff --git a/arch/mips/mm/c-r3k.c b/arch/mips/mm/c-r3k.c index 704dc735a59df..2fcde0c8ea029 100644 --- a/arch/mips/mm/c-r3k.c +++ b/arch/mips/mm/c-r3k.c @@ -26,7 +26,7 @@ static unsigned long icache_size, dcache_size; /* Size in bytes */ static unsigned long icache_lsize, dcache_lsize; /* Size in bytes */ -unsigned long __cpuinit r3k_cache_size(unsigned long ca_flags) +unsigned long r3k_cache_size(unsigned long ca_flags) { unsigned long flags, status, dummy, size; volatile unsigned long *p; @@ -61,7 +61,7 @@ unsigned long __cpuinit r3k_cache_size(unsigned long ca_flags) return size * sizeof(*p); } -unsigned long __cpuinit r3k_cache_lsize(unsigned long ca_flags) +unsigned long r3k_cache_lsize(unsigned long ca_flags) { unsigned long flags, status, lsize, i; volatile unsigned long *p; @@ -90,7 +90,7 @@ unsigned long __cpuinit r3k_cache_lsize(unsigned long ca_flags) return lsize * sizeof(*p); } -static void __cpuinit r3k_probe_cache(void) +static void r3k_probe_cache(void) { dcache_size = r3k_cache_size(ST0_ISC); if (dcache_size) @@ -312,7 +312,7 @@ static void r3k_dma_cache_wback_inv(unsigned long start, unsigned long size) r3k_flush_dcache_range(start, start + size); } -void __cpuinit r3k_cache_init(void) +void r3k_cache_init(void) { extern void build_clear_page(void); extern void build_copy_page(void); diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index c2ec87e5d1cc6..e48a1e5240ee5 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -108,7 +108,7 @@ static inline void r4k_blast_dcache_page_dc64(unsigned long addr) blast_dcache64_page(addr); } -static void __cpuinit r4k_blast_dcache_page_setup(void) +static void r4k_blast_dcache_page_setup(void) { unsigned long dc_lsize = cpu_dcache_line_size(); @@ -124,7 +124,7 @@ static void __cpuinit r4k_blast_dcache_page_setup(void) static void (* r4k_blast_dcache_page_indexed)(unsigned long addr); -static void __cpuinit r4k_blast_dcache_page_indexed_setup(void) +static void r4k_blast_dcache_page_indexed_setup(void) { unsigned long dc_lsize = cpu_dcache_line_size(); @@ -141,7 +141,7 @@ static void __cpuinit r4k_blast_dcache_page_indexed_setup(void) void (* r4k_blast_dcache)(void); EXPORT_SYMBOL(r4k_blast_dcache); -static void __cpuinit r4k_blast_dcache_setup(void) +static void r4k_blast_dcache_setup(void) { unsigned long dc_lsize = cpu_dcache_line_size(); @@ -228,7 +228,7 @@ static inline void tx49_blast_icache32_page_indexed(unsigned long page) static void (* r4k_blast_icache_page)(unsigned long addr); -static void __cpuinit r4k_blast_icache_page_setup(void) +static void r4k_blast_icache_page_setup(void) { unsigned long ic_lsize = cpu_icache_line_size(); @@ -245,7 +245,7 @@ static void __cpuinit r4k_blast_icache_page_setup(void) static void (* r4k_blast_icache_page_indexed)(unsigned long addr); -static void __cpuinit r4k_blast_icache_page_indexed_setup(void) +static void r4k_blast_icache_page_indexed_setup(void) { unsigned long ic_lsize = cpu_icache_line_size(); @@ -270,7 +270,7 @@ static void __cpuinit r4k_blast_icache_page_indexed_setup(void) void (* r4k_blast_icache)(void); EXPORT_SYMBOL(r4k_blast_icache); -static void __cpuinit r4k_blast_icache_setup(void) +static void r4k_blast_icache_setup(void) { unsigned long ic_lsize = cpu_icache_line_size(); @@ -291,7 +291,7 @@ static void __cpuinit r4k_blast_icache_setup(void) static void (* r4k_blast_scache_page)(unsigned long addr); -static void __cpuinit r4k_blast_scache_page_setup(void) +static void r4k_blast_scache_page_setup(void) { unsigned long sc_lsize = cpu_scache_line_size(); @@ -309,7 +309,7 @@ static void __cpuinit r4k_blast_scache_page_setup(void) static void (* r4k_blast_scache_page_indexed)(unsigned long addr); -static void __cpuinit r4k_blast_scache_page_indexed_setup(void) +static void r4k_blast_scache_page_indexed_setup(void) { unsigned long sc_lsize = cpu_scache_line_size(); @@ -327,7 +327,7 @@ static void __cpuinit r4k_blast_scache_page_indexed_setup(void) static void (* r4k_blast_scache)(void); -static void __cpuinit r4k_blast_scache_setup(void) +static void r4k_blast_scache_setup(void) { unsigned long sc_lsize = cpu_scache_line_size(); @@ -804,11 +804,11 @@ static inline void alias_74k_erratum(struct cpuinfo_mips *c) } } -static char *way_string[] __cpuinitdata = { NULL, "direct mapped", "2-way", +static char *way_string[] = { NULL, "direct mapped", "2-way", "3-way", "4-way", "5-way", "6-way", "7-way", "8-way" }; -static void __cpuinit probe_pcache(void) +static void probe_pcache(void) { struct cpuinfo_mips *c = ¤t_cpu_data; unsigned int config = read_c0_config(); @@ -1126,7 +1126,7 @@ static void __cpuinit probe_pcache(void) * executes in KSEG1 space or else you will crash and burn badly. You have * been warned. */ -static int __cpuinit probe_scache(void) +static int probe_scache(void) { unsigned long flags, addr, begin, end, pow2; unsigned int config = read_c0_config(); @@ -1203,7 +1203,7 @@ extern int r5k_sc_init(void); extern int rm7k_sc_init(void); extern int mips_sc_init(void); -static void __cpuinit setup_scache(void) +static void setup_scache(void) { struct cpuinfo_mips *c = ¤t_cpu_data; unsigned int config = read_c0_config(); @@ -1336,7 +1336,7 @@ static void nxp_pr4450_fixup_config(void) NXP_BARRIER(); } -static int __cpuinitdata cca = -1; +static int cca = -1; static int __init cca_setup(char *str) { @@ -1347,7 +1347,7 @@ static int __init cca_setup(char *str) early_param("cca", cca_setup); -static void __cpuinit coherency_setup(void) +static void coherency_setup(void) { if (cca < 0 || cca > 7) cca = read_c0_config() & CONF_CM_CMASK; @@ -1387,7 +1387,7 @@ static void __cpuinit coherency_setup(void) } } -static void __cpuinit r4k_cache_error_setup(void) +static void r4k_cache_error_setup(void) { extern char __weak except_vec2_generic; extern char __weak except_vec2_sb1; @@ -1405,7 +1405,7 @@ static void __cpuinit r4k_cache_error_setup(void) } } -void __cpuinit r4k_cache_init(void) +void r4k_cache_init(void) { extern void build_clear_page(void); extern void build_copy_page(void); diff --git a/arch/mips/mm/c-tx39.c b/arch/mips/mm/c-tx39.c index ba9da270289fa..8d909dbbf37f5 100644 --- a/arch/mips/mm/c-tx39.c +++ b/arch/mips/mm/c-tx39.c @@ -344,7 +344,7 @@ static __init void tx39_probe_cache(void) } } -void __cpuinit tx39_cache_init(void) +void tx39_cache_init(void) { extern void build_clear_page(void); extern void build_copy_page(void); diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c index 5aeb3eb0b72f8..15f813c303b45 100644 --- a/arch/mips/mm/cache.c +++ b/arch/mips/mm/cache.c @@ -182,7 +182,7 @@ static inline void setup_protection_map(void) } } -void __cpuinit cpu_cache_init(void) +void cpu_cache_init(void) { if (cpu_has_3k_cache) { extern void __weak r3k_cache_init(void); diff --git a/arch/mips/mm/page.c b/arch/mips/mm/page.c index 4eb8dcfaf1ce1..e5506a54959cd 100644 --- a/arch/mips/mm/page.c +++ b/arch/mips/mm/page.c @@ -66,8 +66,8 @@ UASM_L_LA(_copy_pref_both) UASM_L_LA(_copy_pref_store) /* We need one branch and therefore one relocation per target label. */ -static struct uasm_label __cpuinitdata labels[5]; -static struct uasm_reloc __cpuinitdata relocs[5]; +static struct uasm_label labels[5]; +static struct uasm_reloc relocs[5]; #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) @@ -88,7 +88,7 @@ static int half_copy_loop_size __cpuinitdata; static int cache_line_size __cpuinitdata; #define cache_line_mask() (cache_line_size - 1) -static inline void __cpuinit +static inline void pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off) { if (cpu_has_64bit_gp_regs && DADDI_WAR && r4k_daddiu_bug()) { @@ -108,7 +108,7 @@ pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off) } } -static void __cpuinit set_prefetch_parameters(void) +static void set_prefetch_parameters(void) { if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) clear_word_size = 8; @@ -199,7 +199,7 @@ static void __cpuinit set_prefetch_parameters(void) 4 * copy_word_size)); } -static void __cpuinit build_clear_store(u32 **buf, int off) +static void build_clear_store(u32 **buf, int off) { if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) { uasm_i_sd(buf, ZERO, off, A0); @@ -208,7 +208,7 @@ static void __cpuinit build_clear_store(u32 **buf, int off) } } -static inline void __cpuinit build_clear_pref(u32 **buf, int off) +static inline void build_clear_pref(u32 **buf, int off) { if (off & cache_line_mask()) return; @@ -240,7 +240,7 @@ extern u32 __clear_page_end; extern u32 __copy_page_start; extern u32 __copy_page_end; -void __cpuinit build_clear_page(void) +void build_clear_page(void) { int off; u32 *buf = &__clear_page_start; @@ -333,7 +333,7 @@ void __cpuinit build_clear_page(void) pr_debug("\t.set pop\n"); } -static void __cpuinit build_copy_load(u32 **buf, int reg, int off) +static void build_copy_load(u32 **buf, int reg, int off) { if (cpu_has_64bit_gp_regs) { uasm_i_ld(buf, reg, off, A1); @@ -342,7 +342,7 @@ static void __cpuinit build_copy_load(u32 **buf, int reg, int off) } } -static void __cpuinit build_copy_store(u32 **buf, int reg, int off) +static void build_copy_store(u32 **buf, int reg, int off) { if (cpu_has_64bit_gp_regs) { uasm_i_sd(buf, reg, off, A0); @@ -387,7 +387,7 @@ static inline void build_copy_store_pref(u32 **buf, int off) } } -void __cpuinit build_copy_page(void) +void build_copy_page(void) { int off; u32 *buf = &__copy_page_start; diff --git a/arch/mips/mm/sc-ip22.c b/arch/mips/mm/sc-ip22.c index c6aaed934d536..dc7c5a5214a9e 100644 --- a/arch/mips/mm/sc-ip22.c +++ b/arch/mips/mm/sc-ip22.c @@ -167,7 +167,7 @@ static struct bcache_ops indy_sc_ops = { .bc_inv = indy_sc_wback_invalidate }; -void __cpuinit indy_sc_init(void) +void indy_sc_init(void) { if (indy_sc_probe()) { indy_sc_enable(); diff --git a/arch/mips/mm/sc-mips.c b/arch/mips/mm/sc-mips.c index df96da7e939b5..5d01392e3518d 100644 --- a/arch/mips/mm/sc-mips.c +++ b/arch/mips/mm/sc-mips.c @@ -132,7 +132,7 @@ static inline int __init mips_sc_probe(void) return 1; } -int __cpuinit mips_sc_init(void) +int mips_sc_init(void) { int found = mips_sc_probe(); if (found) { diff --git a/arch/mips/mm/sc-r5k.c b/arch/mips/mm/sc-r5k.c index 8bc67720e145f..0216ed6eaa2a9 100644 --- a/arch/mips/mm/sc-r5k.c +++ b/arch/mips/mm/sc-r5k.c @@ -98,7 +98,7 @@ static struct bcache_ops r5k_sc_ops = { .bc_inv = r5k_dma_cache_inv_sc }; -void __cpuinit r5k_sc_init(void) +void r5k_sc_init(void) { if (r5k_sc_probe()) { r5k_sc_enable(); diff --git a/arch/mips/mm/sc-rm7k.c b/arch/mips/mm/sc-rm7k.c index 274af3be1442b..aaffbba337064 100644 --- a/arch/mips/mm/sc-rm7k.c +++ b/arch/mips/mm/sc-rm7k.c @@ -104,7 +104,7 @@ static void blast_rm7k_tcache(void) /* * This function is executed in uncached address space. */ -static __cpuinit void __rm7k_tc_enable(void) +static void __rm7k_tc_enable(void) { int i; @@ -117,7 +117,7 @@ static __cpuinit void __rm7k_tc_enable(void) cache_op(Index_Store_Tag_T, CKSEG0ADDR(i)); } -static __cpuinit void rm7k_tc_enable(void) +static void rm7k_tc_enable(void) { if (read_c0_config() & RM7K_CONF_TE) return; @@ -130,7 +130,7 @@ static __cpuinit void rm7k_tc_enable(void) /* * This function is executed in uncached address space. */ -static __cpuinit void __rm7k_sc_enable(void) +static void __rm7k_sc_enable(void) { int i; @@ -143,7 +143,7 @@ static __cpuinit void __rm7k_sc_enable(void) cache_op(Index_Store_Tag_SD, CKSEG0ADDR(i)); } -static __cpuinit void rm7k_sc_enable(void) +static void rm7k_sc_enable(void) { if (read_c0_config() & RM7K_CONF_SE) return; @@ -184,7 +184,7 @@ static struct bcache_ops rm7k_sc_ops = { * This is a probing function like the one found in c-r4k.c, we look for the * wrap around point with different addresses. */ -static __cpuinit void __probe_tcache(void) +static void __probe_tcache(void) { unsigned long flags, addr, begin, end, pow2; @@ -226,7 +226,7 @@ static __cpuinit void __probe_tcache(void) local_irq_restore(flags); } -void __cpuinit rm7k_sc_init(void) +void rm7k_sc_init(void) { struct cpuinfo_mips *c = ¤t_cpu_data; unsigned int config = read_c0_config(); diff --git a/arch/mips/mm/tlb-r3k.c b/arch/mips/mm/tlb-r3k.c index a63d1ed0827fe..9aca10994cd22 100644 --- a/arch/mips/mm/tlb-r3k.c +++ b/arch/mips/mm/tlb-r3k.c @@ -276,7 +276,7 @@ void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, } } -void __cpuinit tlb_init(void) +void tlb_init(void) { local_flush_tlb_all(); diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c index c643de4c473a8..00b26a67a06d8 100644 --- a/arch/mips/mm/tlb-r4k.c +++ b/arch/mips/mm/tlb-r4k.c @@ -389,7 +389,7 @@ int __init has_transparent_hugepage(void) #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -static int __cpuinitdata ntlb; +static int ntlb; static int __init set_ntlb(char *str) { get_option(&str, &ntlb); @@ -398,7 +398,7 @@ static int __init set_ntlb(char *str) __setup("ntlb=", set_ntlb); -void __cpuinit tlb_init(void) +void tlb_init(void) { /* * You should never change this register: diff --git a/arch/mips/mm/tlb-r8k.c b/arch/mips/mm/tlb-r8k.c index 91c2499f806a2..6a99733a44402 100644 --- a/arch/mips/mm/tlb-r8k.c +++ b/arch/mips/mm/tlb-r8k.c @@ -213,14 +213,14 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) local_irq_restore(flags); } -static void __cpuinit probe_tlb(unsigned long config) +static void probe_tlb(unsigned long config) { struct cpuinfo_mips *c = ¤t_cpu_data; c->tlbsize = 3 * 128; /* 3 sets each 128 entries */ } -void __cpuinit tlb_init(void) +void tlb_init(void) { unsigned int config = read_c0_config(); unsigned long status; diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index a91a7a99f70f3..842b087443a8b 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -136,7 +136,7 @@ static int scratchpad_offset(int i) * why; it's not an issue caused by the core RTL. * */ -static int __cpuinit m4kc_tlbp_war(void) +static int m4kc_tlbp_war(void) { return (current_cpu_data.processor_id & 0xffff00) == (PRID_COMP_MIPS | PRID_IMP_4KC); @@ -181,9 +181,9 @@ UASM_L_LA(_large_segbits_fault) UASM_L_LA(_tlb_huge_update) #endif -static int __cpuinitdata hazard_instance; +static int hazard_instance; -static void __cpuinit uasm_bgezl_hazard(u32 **p, +static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance) { @@ -196,7 +196,7 @@ static void __cpuinit uasm_bgezl_hazard(u32 **p, } } -static void __cpuinit uasm_bgezl_label(struct uasm_label **l, +static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance) { @@ -305,7 +305,7 @@ static int check_for_high_segbits __cpuinitdata; static unsigned int kscratch_used_mask __cpuinitdata; -static int __cpuinit allocate_kscratch(void) +static int allocate_kscratch(void) { int r; unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask; @@ -326,7 +326,7 @@ static int scratch_reg __cpuinitdata; static int pgd_reg __cpuinitdata; enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch}; -static struct work_registers __cpuinit build_get_work_registers(u32 **p) +static struct work_registers build_get_work_registers(u32 **p) { struct work_registers r; @@ -382,7 +382,7 @@ static struct work_registers __cpuinit build_get_work_registers(u32 **p) return r; } -static void __cpuinit build_restore_work_registers(u32 **p) +static void build_restore_work_registers(u32 **p) { if (scratch_reg > 0) { UASM_i_MFC0(p, 1, 31, scratch_reg); @@ -407,7 +407,7 @@ extern unsigned long pgd_current[]; /* * The R3000 TLB handler is simple. */ -static void __cpuinit build_r3000_tlb_refill_handler(void) +static void build_r3000_tlb_refill_handler(void) { long pgdc = (long)pgd_current; u32 *p; @@ -476,7 +476,7 @@ static u32 final_handler[64] __cpuinitdata; * * As if we MIPS hackers wouldn't know how to nop pipelines happy ... */ -static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p) +static void __maybe_unused build_tlb_probe_entry(u32 **p) { switch (current_cpu_type()) { /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ @@ -500,7 +500,7 @@ static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p) */ enum tlb_write_entry { tlb_random, tlb_indexed }; -static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, +static void build_tlb_write_entry(u32 **p, struct uasm_label **l, struct uasm_reloc **r, enum tlb_write_entry wmode) { @@ -636,7 +636,7 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, } } -static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p, +static __maybe_unused void build_convert_pte_to_entrylo(u32 **p, unsigned int reg) { if (cpu_has_rixi) { @@ -652,7 +652,7 @@ static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p, #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT -static __cpuinit void build_restore_pagemask(u32 **p, +static void build_restore_pagemask(u32 **p, struct uasm_reloc **r, unsigned int tmp, enum label_id lid, @@ -695,7 +695,7 @@ static __cpuinit void build_restore_pagemask(u32 **p, } } -static __cpuinit void build_huge_tlb_write_entry(u32 **p, +static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int tmp, @@ -715,7 +715,7 @@ static __cpuinit void build_huge_tlb_write_entry(u32 **p, /* * Check if Huge PTE is present, if so then jump to LABEL. */ -static void __cpuinit +static void build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, unsigned int pmd, int lid) { @@ -728,7 +728,7 @@ build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, } } -static __cpuinit void build_huge_update_entries(u32 **p, +static void build_huge_update_entries(u32 **p, unsigned int pte, unsigned int tmp) { @@ -760,7 +760,7 @@ static __cpuinit void build_huge_update_entries(u32 **p, UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ } -static __cpuinit void build_huge_handler_tail(u32 **p, +static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r, struct uasm_label **l, unsigned int pte, @@ -783,7 +783,7 @@ static __cpuinit void build_huge_handler_tail(u32 **p, * TMP and PTR are scratch. * TMP will be clobbered, PTR will hold the pmd entry. */ -static void __cpuinit +static void build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int tmp, unsigned int ptr) { @@ -875,7 +875,7 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, * BVADDR is the faulting address, PTR is scratch. * PTR will hold the pgd for vmalloc. */ -static void __cpuinit +static void build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int bvaddr, unsigned int ptr, enum vmalloc64_mode mode) @@ -945,7 +945,7 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, * TMP and PTR are scratch. * TMP will be clobbered, PTR will hold the pgd entry. */ -static void __cpuinit __maybe_unused +static void __maybe_unused build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) { long pgdc = (long)pgd_current; @@ -980,7 +980,7 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) #endif /* !CONFIG_64BIT */ -static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx) +static void build_adjust_context(u32 **p, unsigned int ctx) { unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); @@ -1006,7 +1006,7 @@ static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx) uasm_i_andi(p, ctx, ctx, mask); } -static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) +static void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) { /* * Bug workaround for the Nevada. It seems as if under certain @@ -1031,7 +1031,7 @@ static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ } -static void __cpuinit build_update_entries(u32 **p, unsigned int tmp, +static void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep) { /* @@ -1094,7 +1094,7 @@ struct mips_huge_tlb_info { bool need_reload_pte; }; -static struct mips_huge_tlb_info __cpuinit +static struct mips_huge_tlb_info build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int tmp, unsigned int ptr, int c0_scratch) @@ -1273,7 +1273,7 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, */ #define MIPS64_REFILL_INSNS 32 -static void __cpuinit build_r4000_tlb_refill_handler(void) +static void build_r4000_tlb_refill_handler(void) { u32 *p = tlb_handler; struct uasm_label *l = labels; @@ -1461,7 +1461,7 @@ u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned; #ifdef CONFIG_MIPS_PGD_C0_CONTEXT u32 tlbmiss_handler_setup_pgd_array[16] __cacheline_aligned; -static void __cpuinit build_r4000_setup_pgd(void) +static void build_r4000_setup_pgd(void) { const int a0 = 4; const int a1 = 5; @@ -1509,7 +1509,7 @@ static void __cpuinit build_r4000_setup_pgd(void) } #endif -static void __cpuinit +static void iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) { #ifdef CONFIG_SMP @@ -1529,7 +1529,7 @@ iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) #endif } -static void __cpuinit +static void iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, unsigned int mode) { @@ -1589,7 +1589,7 @@ iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, * the page table where this PTE is located, PTE will be re-loaded * with it's original value. */ -static void __cpuinit +static void build_pte_present(u32 **p, struct uasm_reloc **r, int pte, int ptr, int scratch, enum label_id lid) { @@ -1617,7 +1617,7 @@ build_pte_present(u32 **p, struct uasm_reloc **r, } /* Make PTE valid, store result in PTR. */ -static void __cpuinit +static void build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr) { @@ -1630,7 +1630,7 @@ build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, * Check if PTE can be written to, if not branch to LABEL. Regardless * restore PTE with value from PTR when done. */ -static void __cpuinit +static void build_pte_writable(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, int scratch, enum label_id lid) @@ -1650,7 +1650,7 @@ build_pte_writable(u32 **p, struct uasm_reloc **r, /* Make PTE writable, update software status bits as well, then store * at PTR. */ -static void __cpuinit +static void build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr) { @@ -1664,7 +1664,7 @@ build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, * Check if PTE can be modified, if not branch to LABEL. Regardless * restore PTE with value from PTR when done. */ -static void __cpuinit +static void build_pte_modifiable(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, int scratch, enum label_id lid) @@ -1693,7 +1693,7 @@ build_pte_modifiable(u32 **p, struct uasm_reloc **r, * This places the pte into ENTRYLO0 and writes it with tlbwi. * Then it returns. */ -static void __cpuinit +static void build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) { uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ @@ -1709,7 +1709,7 @@ build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) * may have the probe fail bit set as a result of a trap on a * kseg2 access, i.e. without refill. Then it returns. */ -static void __cpuinit +static void build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int pte, unsigned int tmp) @@ -1727,7 +1727,7 @@ build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, uasm_i_rfe(p); /* branch delay */ } -static void __cpuinit +static void build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, unsigned int ptr) { @@ -1747,7 +1747,7 @@ build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, uasm_i_tlbp(p); /* load delay */ } -static void __cpuinit build_r3000_tlb_load_handler(void) +static void build_r3000_tlb_load_handler(void) { u32 *p = handle_tlbl; struct uasm_label *l = labels; @@ -1777,7 +1777,7 @@ static void __cpuinit build_r3000_tlb_load_handler(void) dump_handler("r3000_tlb_load", handle_tlbl, ARRAY_SIZE(handle_tlbl)); } -static void __cpuinit build_r3000_tlb_store_handler(void) +static void build_r3000_tlb_store_handler(void) { u32 *p = handle_tlbs; struct uasm_label *l = labels; @@ -1807,7 +1807,7 @@ static void __cpuinit build_r3000_tlb_store_handler(void) dump_handler("r3000_tlb_store", handle_tlbs, ARRAY_SIZE(handle_tlbs)); } -static void __cpuinit build_r3000_tlb_modify_handler(void) +static void build_r3000_tlb_modify_handler(void) { u32 *p = handle_tlbm; struct uasm_label *l = labels; @@ -1841,7 +1841,7 @@ static void __cpuinit build_r3000_tlb_modify_handler(void) /* * R4000 style TLB load/store/modify handlers. */ -static struct work_registers __cpuinit +static struct work_registers build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, struct uasm_reloc **r) { @@ -1877,7 +1877,7 @@ build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, return wr; } -static void __cpuinit +static void build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int tmp, unsigned int ptr) @@ -1895,7 +1895,7 @@ build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, #endif } -static void __cpuinit build_r4000_tlb_load_handler(void) +static void build_r4000_tlb_load_handler(void) { u32 *p = handle_tlbl; struct uasm_label *l = labels; @@ -2051,7 +2051,7 @@ static void __cpuinit build_r4000_tlb_load_handler(void) dump_handler("r4000_tlb_load", handle_tlbl, ARRAY_SIZE(handle_tlbl)); } -static void __cpuinit build_r4000_tlb_store_handler(void) +static void build_r4000_tlb_store_handler(void) { u32 *p = handle_tlbs; struct uasm_label *l = labels; @@ -2105,7 +2105,7 @@ static void __cpuinit build_r4000_tlb_store_handler(void) dump_handler("r4000_tlb_store", handle_tlbs, ARRAY_SIZE(handle_tlbs)); } -static void __cpuinit build_r4000_tlb_modify_handler(void) +static void build_r4000_tlb_modify_handler(void) { u32 *p = handle_tlbm; struct uasm_label *l = labels; @@ -2160,7 +2160,7 @@ static void __cpuinit build_r4000_tlb_modify_handler(void) dump_handler("r4000_tlb_modify", handle_tlbm, ARRAY_SIZE(handle_tlbm)); } -void __cpuinit build_tlb_refill_handler(void) +void build_tlb_refill_handler(void) { /* * The refill handler is generated per-CPU, multi-node systems @@ -2226,7 +2226,7 @@ void __cpuinit build_tlb_refill_handler(void) } } -void __cpuinit flush_tlb_handlers(void) +void flush_tlb_handlers(void) { local_flush_icache_range((unsigned long)handle_tlbl, (unsigned long)handle_tlbl + sizeof(handle_tlbl)); diff --git a/arch/mips/mti-malta/malta-smtc.c b/arch/mips/mti-malta/malta-smtc.c index becbf47506a59..c4849904f0139 100644 --- a/arch/mips/mti-malta/malta-smtc.c +++ b/arch/mips/mti-malta/malta-smtc.c @@ -32,7 +32,7 @@ static void msmtc_send_ipi_mask(const struct cpumask *mask, unsigned int action) /* * Post-config but pre-boot cleanup entry point */ -static void __cpuinit msmtc_init_secondary(void) +static void msmtc_init_secondary(void) { int myvpe; @@ -53,7 +53,7 @@ static void __cpuinit msmtc_init_secondary(void) /* * Platform "CPU" startup hook */ -static void __cpuinit msmtc_boot_secondary(int cpu, struct task_struct *idle) +static void msmtc_boot_secondary(int cpu, struct task_struct *idle) { smtc_boot_secondary(cpu, idle); } @@ -61,7 +61,7 @@ static void __cpuinit msmtc_boot_secondary(int cpu, struct task_struct *idle) /* * SMP initialization finalization entry point */ -static void __cpuinit msmtc_smp_finish(void) +static void msmtc_smp_finish(void) { smtc_smp_finish(); } diff --git a/arch/mips/mti-malta/malta-time.c b/arch/mips/mti-malta/malta-time.c index 0ad305f75802b..53aad4a353756 100644 --- a/arch/mips/mti-malta/malta-time.c +++ b/arch/mips/mti-malta/malta-time.c @@ -150,7 +150,7 @@ static void __init plat_perf_setup(void) } } -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { #ifdef MSC01E_INT_BASE if (cpu_has_veic) { diff --git a/arch/mips/mti-sead3/sead3-time.c b/arch/mips/mti-sead3/sead3-time.c index 96b42eb9b5e22..a43ea3cc0a3bf 100644 --- a/arch/mips/mti-sead3/sead3-time.c +++ b/arch/mips/mti-sead3/sead3-time.c @@ -91,7 +91,7 @@ static void __init plat_perf_setup(void) } } -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { if (cpu_has_vint) set_vi_handler(cp0_compare_irq, mips_timer_dispatch); diff --git a/arch/mips/netlogic/common/smp.c b/arch/mips/netlogic/common/smp.c index ffba52489bef7..c01d554f64e14 100644 --- a/arch/mips/netlogic/common/smp.c +++ b/arch/mips/netlogic/common/smp.c @@ -116,7 +116,7 @@ void nlm_early_init_secondary(int cpu) /* * Code to run on secondary just after probing the CPU */ -static void __cpuinit nlm_init_secondary(void) +static void nlm_init_secondary(void) { int hwtid; @@ -252,7 +252,7 @@ static int nlm_parse_cpumask(cpumask_t *wakeup_mask) return 0; } -int __cpuinit nlm_wakeup_secondary_cpus(void) +int nlm_wakeup_secondary_cpus(void) { unsigned long reset_vec; char *reset_data; diff --git a/arch/mips/netlogic/common/time.c b/arch/mips/netlogic/common/time.c index 5c56555380bb1..045a396c57ced 100644 --- a/arch/mips/netlogic/common/time.c +++ b/arch/mips/netlogic/common/time.c @@ -54,7 +54,7 @@ #error "Unknown CPU" #endif -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { return IRQ_TIMER; } diff --git a/arch/mips/netlogic/xlr/wakeup.c b/arch/mips/netlogic/xlr/wakeup.c index 3ebf7411d67b1..37a40927ed719 100644 --- a/arch/mips/netlogic/xlr/wakeup.c +++ b/arch/mips/netlogic/xlr/wakeup.c @@ -49,7 +49,7 @@ #include #include -int __cpuinit xlr_wakeup_secondary_cpus(void) +int xlr_wakeup_secondary_cpus(void) { struct nlm_soc_info *nodep; unsigned int i, j, boot_cpu; diff --git a/arch/mips/pci/pci-ip27.c b/arch/mips/pci/pci-ip27.c index 6eb65e44d9e4a..2946bdf654389 100644 --- a/arch/mips/pci/pci-ip27.c +++ b/arch/mips/pci/pci-ip27.c @@ -42,7 +42,7 @@ int irq_to_slot[MAX_PCI_BUSSES * MAX_DEVICES_PER_PCIBUS]; extern struct pci_ops bridge_pci_ops; -int __cpuinit bridge_probe(nasid_t nasid, int widget_id, int masterwid) +int bridge_probe(nasid_t nasid, int widget_id, int masterwid) { unsigned long offset = NODE_OFFSET(nasid); struct bridge_controller *bc; diff --git a/arch/mips/pmcs-msp71xx/msp_smtc.c b/arch/mips/pmcs-msp71xx/msp_smtc.c index c8dcc1c01e181..6aa91401ddfc3 100644 --- a/arch/mips/pmcs-msp71xx/msp_smtc.c +++ b/arch/mips/pmcs-msp71xx/msp_smtc.c @@ -33,7 +33,7 @@ static void msp_smtc_send_ipi_mask(const struct cpumask *mask, /* * Post-config but pre-boot cleanup entry point */ -static void __cpuinit msp_smtc_init_secondary(void) +static void msp_smtc_init_secondary(void) { int myvpe; @@ -48,7 +48,7 @@ static void __cpuinit msp_smtc_init_secondary(void) /* * Platform "CPU" startup hook */ -static void __cpuinit msp_smtc_boot_secondary(int cpu, +static void msp_smtc_boot_secondary(int cpu, struct task_struct *idle) { smtc_boot_secondary(cpu, idle); @@ -57,7 +57,7 @@ static void __cpuinit msp_smtc_boot_secondary(int cpu, /* * SMP initialization finalization entry point */ -static void __cpuinit msp_smtc_smp_finish(void) +static void msp_smtc_smp_finish(void) { smtc_smp_finish(); } diff --git a/arch/mips/pmcs-msp71xx/msp_time.c b/arch/mips/pmcs-msp71xx/msp_time.c index 8f12ecc55ace5..fea917be0ff10 100644 --- a/arch/mips/pmcs-msp71xx/msp_time.c +++ b/arch/mips/pmcs-msp71xx/msp_time.c @@ -88,7 +88,7 @@ void __init plat_time_init(void) mips_hpt_frequency = cpu_rate/2; } -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { /* MIPS_MT modes may want timer for second VPE */ if ((get_current_vpe()) && !tim_installed) { diff --git a/arch/mips/pnx833x/common/interrupts.c b/arch/mips/pnx833x/common/interrupts.c index a4a90596c0ad4..e460865873c1c 100644 --- a/arch/mips/pnx833x/common/interrupts.c +++ b/arch/mips/pnx833x/common/interrupts.c @@ -281,7 +281,7 @@ void __init arch_init_irq(void) write_c0_status(read_c0_status() | IE_IRQ2); } -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { if (cpu_has_vint) set_vi_handler(cp0_compare_irq, pnx833x_timer_dispatch); diff --git a/arch/mips/powertv/time.c b/arch/mips/powertv/time.c index 9fd7b67f2af7c..f38b0d45eca90 100644 --- a/arch/mips/powertv/time.c +++ b/arch/mips/powertv/time.c @@ -25,7 +25,7 @@ #include "powertv-clock.h" -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { return irq_mips_timer; } diff --git a/arch/mips/ralink/irq.c b/arch/mips/ralink/irq.c index 320b1f1043fff..781b3d14a489e 100644 --- a/arch/mips/ralink/irq.c +++ b/arch/mips/ralink/irq.c @@ -73,7 +73,7 @@ static struct irq_chip ralink_intc_irq_chip = { .irq_mask_ack = ralink_intc_irq_mask, }; -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { return CP0_LEGACY_COMPARE_IRQ; } diff --git a/arch/mips/sgi-ip27/ip27-init.c b/arch/mips/sgi-ip27/ip27-init.c index d41b1c6fb0329..ee736bd103f85 100644 --- a/arch/mips/sgi-ip27/ip27-init.c +++ b/arch/mips/sgi-ip27/ip27-init.c @@ -54,7 +54,7 @@ extern void pcibr_setup(cnodeid_t); extern void xtalk_probe_node(cnodeid_t nid); -static void __cpuinit per_hub_init(cnodeid_t cnode) +static void per_hub_init(cnodeid_t cnode) { struct hub_data *hub = hub_data(cnode); nasid_t nasid = COMPACT_TO_NASID_NODEID(cnode); @@ -110,7 +110,7 @@ static void __cpuinit per_hub_init(cnodeid_t cnode) } } -void __cpuinit per_cpu_init(void) +void per_cpu_init(void) { int cpu = smp_processor_id(); int slice = LOCAL_HUB_L(PI_CPU_NUM); diff --git a/arch/mips/sgi-ip27/ip27-smp.c b/arch/mips/sgi-ip27/ip27-smp.c index f94638141b20e..f4ea8aa79ba2d 100644 --- a/arch/mips/sgi-ip27/ip27-smp.c +++ b/arch/mips/sgi-ip27/ip27-smp.c @@ -173,12 +173,12 @@ static void ip27_send_ipi_mask(const struct cpumask *mask, unsigned int action) ip27_send_ipi_single(i, action); } -static void __cpuinit ip27_init_secondary(void) +static void ip27_init_secondary(void) { per_cpu_init(); } -static void __cpuinit ip27_smp_finish(void) +static void ip27_smp_finish(void) { extern void hub_rt_clock_event_init(void); @@ -195,7 +195,7 @@ static void __init ip27_cpus_done(void) * set sp to the kernel stack of the newly created idle process, gp to the proc * struct so that current_thread_info() will work. */ -static void __cpuinit ip27_boot_secondary(int cpu, struct task_struct *idle) +static void ip27_boot_secondary(int cpu, struct task_struct *idle) { unsigned long gp = (unsigned long)task_thread_info(idle); unsigned long sp = __KSTK_TOS(idle); diff --git a/arch/mips/sgi-ip27/ip27-timer.c b/arch/mips/sgi-ip27/ip27-timer.c index 2e21b761cb9c7..1d97eaba0c5f0 100644 --- a/arch/mips/sgi-ip27/ip27-timer.c +++ b/arch/mips/sgi-ip27/ip27-timer.c @@ -106,7 +106,7 @@ struct irqaction hub_rt_irqaction = { #define NSEC_PER_CYCLE 800 #define CYCLES_PER_SEC (NSEC_PER_SEC / NSEC_PER_CYCLE) -void __cpuinit hub_rt_clock_event_init(void) +void hub_rt_clock_event_init(void) { unsigned int cpu = smp_processor_id(); struct clock_event_device *cd = &per_cpu(hub_rt_clockevent, cpu); @@ -173,7 +173,7 @@ void __init plat_time_init(void) hub_rt_clock_event_init(); } -void __cpuinit cpu_time_init(void) +void cpu_time_init(void) { lboard_t *board; klcpu_t *cpu; @@ -194,7 +194,7 @@ void __cpuinit cpu_time_init(void) set_c0_status(SRB_TIMOCLK); } -void __cpuinit hub_rtc_init(cnodeid_t cnode) +void hub_rtc_init(cnodeid_t cnode) { /* diff --git a/arch/mips/sgi-ip27/ip27-xtalk.c b/arch/mips/sgi-ip27/ip27-xtalk.c index a4df7d0f6f12d..d59b820f528d1 100644 --- a/arch/mips/sgi-ip27/ip27-xtalk.c +++ b/arch/mips/sgi-ip27/ip27-xtalk.c @@ -23,7 +23,7 @@ extern int bridge_probe(nasid_t nasid, int widget, int masterwid); -static int __cpuinit probe_one_port(nasid_t nasid, int widget, int masterwid) +static int probe_one_port(nasid_t nasid, int widget, int masterwid) { widgetreg_t widget_id; xwidget_part_num_t partnum; @@ -47,7 +47,7 @@ static int __cpuinit probe_one_port(nasid_t nasid, int widget, int masterwid) return 0; } -static int __cpuinit xbow_probe(nasid_t nasid) +static int xbow_probe(nasid_t nasid) { lboard_t *brd; klxbow_t *xbow_p; @@ -100,7 +100,7 @@ static int __cpuinit xbow_probe(nasid_t nasid) return 0; } -void __cpuinit xtalk_probe_node(cnodeid_t nid) +void xtalk_probe_node(cnodeid_t nid) { volatile u64 hubreg; nasid_t nasid; diff --git a/arch/mips/sibyte/bcm1480/smp.c b/arch/mips/sibyte/bcm1480/smp.c index de88e22694a05..54e2c4de15c16 100644 --- a/arch/mips/sibyte/bcm1480/smp.c +++ b/arch/mips/sibyte/bcm1480/smp.c @@ -60,7 +60,7 @@ static void *mailbox_0_regs[] = { /* * SMP init and finish on secondary CPUs */ -void __cpuinit bcm1480_smp_init(void) +void bcm1480_smp_init(void) { unsigned int imask = STATUSF_IP4 | STATUSF_IP3 | STATUSF_IP2 | STATUSF_IP1 | STATUSF_IP0; @@ -95,7 +95,7 @@ static void bcm1480_send_ipi_mask(const struct cpumask *mask, /* * Code to run on secondary just after probing the CPU */ -static void __cpuinit bcm1480_init_secondary(void) +static void bcm1480_init_secondary(void) { extern void bcm1480_smp_init(void); @@ -106,7 +106,7 @@ static void __cpuinit bcm1480_init_secondary(void) * Do any tidying up before marking online and running the idle * loop */ -static void __cpuinit bcm1480_smp_finish(void) +static void bcm1480_smp_finish(void) { extern void sb1480_clockevent_init(void); @@ -125,7 +125,7 @@ static void bcm1480_cpus_done(void) * Setup the PC, SP, and GP of a secondary processor and start it * running! */ -static void __cpuinit bcm1480_boot_secondary(int cpu, struct task_struct *idle) +static void bcm1480_boot_secondary(int cpu, struct task_struct *idle) { int retval; diff --git a/arch/mips/sibyte/sb1250/smp.c b/arch/mips/sibyte/sb1250/smp.c index 285cfef4ebc08..d7b942db0ea51 100644 --- a/arch/mips/sibyte/sb1250/smp.c +++ b/arch/mips/sibyte/sb1250/smp.c @@ -48,7 +48,7 @@ static void *mailbox_regs[] = { /* * SMP init and finish on secondary CPUs */ -void __cpuinit sb1250_smp_init(void) +void sb1250_smp_init(void) { unsigned int imask = STATUSF_IP4 | STATUSF_IP3 | STATUSF_IP2 | STATUSF_IP1 | STATUSF_IP0; @@ -83,7 +83,7 @@ static inline void sb1250_send_ipi_mask(const struct cpumask *mask, /* * Code to run on secondary just after probing the CPU */ -static void __cpuinit sb1250_init_secondary(void) +static void sb1250_init_secondary(void) { extern void sb1250_smp_init(void); @@ -94,7 +94,7 @@ static void __cpuinit sb1250_init_secondary(void) * Do any tidying up before marking online and running the idle * loop */ -static void __cpuinit sb1250_smp_finish(void) +static void sb1250_smp_finish(void) { extern void sb1250_clockevent_init(void); @@ -113,7 +113,7 @@ static void sb1250_cpus_done(void) * Setup the PC, SP, and GP of a secondary processor and start it * running! */ -static void __cpuinit sb1250_boot_secondary(int cpu, struct task_struct *idle) +static void sb1250_boot_secondary(int cpu, struct task_struct *idle) { int retval; diff --git a/arch/openrisc/kernel/setup.c b/arch/openrisc/kernel/setup.c index f4d5bedc3b4f7..d7359ffbcbdd3 100644 --- a/arch/openrisc/kernel/setup.c +++ b/arch/openrisc/kernel/setup.c @@ -267,7 +267,7 @@ void __init detect_unit_config(unsigned long upr, unsigned long mask, * */ -void __cpuinit calibrate_delay(void) +void calibrate_delay(void) { const int *val; struct device_node *cpu = NULL; diff --git a/arch/parisc/kernel/firmware.c b/arch/parisc/kernel/firmware.c index f65fa480c905f..4e6a60470bce8 100644 --- a/arch/parisc/kernel/firmware.c +++ b/arch/parisc/kernel/firmware.c @@ -150,7 +150,7 @@ static void convert_to_wide(unsigned long *addr) } #ifdef CONFIG_64BIT -void __cpuinit set_firmware_width_unlocked(void) +void set_firmware_width_unlocked(void) { int ret; @@ -167,7 +167,7 @@ void __cpuinit set_firmware_width_unlocked(void) * This function must be called before any pdc_* function that uses the * convert_to_wide function. */ -void __cpuinit set_firmware_width(void) +void set_firmware_width(void) { unsigned long flags; spin_lock_irqsave(&pdc_lock, flags); @@ -175,11 +175,11 @@ void __cpuinit set_firmware_width(void) spin_unlock_irqrestore(&pdc_lock, flags); } #else -void __cpuinit set_firmware_width_unlocked(void) { +void set_firmware_width_unlocked(void) { return; } -void __cpuinit set_firmware_width(void) { +void set_firmware_width(void) { return; } #endif /*CONFIG_64BIT*/ @@ -301,7 +301,7 @@ int pdc_chassis_warn(unsigned long *warn) return retval; } -int __cpuinit pdc_coproc_cfg_unlocked(struct pdc_coproc_cfg *pdc_coproc_info) +int pdc_coproc_cfg_unlocked(struct pdc_coproc_cfg *pdc_coproc_info) { int ret; @@ -322,7 +322,7 @@ int __cpuinit pdc_coproc_cfg_unlocked(struct pdc_coproc_cfg *pdc_coproc_info) * This PDC call returns the presence and status of all the coprocessors * attached to the processor. */ -int __cpuinit pdc_coproc_cfg(struct pdc_coproc_cfg *pdc_coproc_info) +int pdc_coproc_cfg(struct pdc_coproc_cfg *pdc_coproc_info) { int ret; unsigned long flags; diff --git a/arch/parisc/kernel/hardware.c b/arch/parisc/kernel/hardware.c index c22c3d84e28b5..5b2c5f3940358 100644 --- a/arch/parisc/kernel/hardware.c +++ b/arch/parisc/kernel/hardware.c @@ -1368,7 +1368,7 @@ const char *parisc_hardware_description(struct parisc_device_id *id) /* Interpret hversion (ret[0]) from PDC_MODEL(4)/PDC_MODEL_INFO(0) */ -enum cpu_type __cpuinit +enum cpu_type parisc_get_cpu_type(unsigned long hversion) { struct hp_cpu_type_mask *ptr; diff --git a/arch/parisc/kernel/processor.c b/arch/parisc/kernel/processor.c index c8fb61ed32f4e..a128a94204e68 100644 --- a/arch/parisc/kernel/processor.c +++ b/arch/parisc/kernel/processor.c @@ -73,7 +73,7 @@ extern int update_cr16_clocksource(void); /* from time.c */ * * FIXME: doesn't do much yet... */ -static void __cpuinit +static void init_percpu_prof(unsigned long cpunum) { struct cpuinfo_parisc *p; @@ -92,7 +92,7 @@ init_percpu_prof(unsigned long cpunum) * (return 1). If so, initialize the chip and tell other partners in crime * they have work to do. */ -static int __cpuinit processor_probe(struct parisc_device *dev) +static int processor_probe(struct parisc_device *dev) { unsigned long txn_addr; unsigned long cpuid; @@ -299,7 +299,7 @@ void __init collect_boot_cpu_data(void) * * o Enable CPU profiling hooks. */ -int __cpuinit init_per_cpu(int cpunum) +int init_per_cpu(int cpunum) { int ret; struct pdc_coproc_cfg coproc_cfg; diff --git a/arch/parisc/kernel/smp.c b/arch/parisc/kernel/smp.c index e3614fb343e5e..71cb54ab9eaa5 100644 --- a/arch/parisc/kernel/smp.c +++ b/arch/parisc/kernel/smp.c @@ -64,7 +64,7 @@ volatile struct task_struct *smp_init_current_idle_task; /* track which CPU is booting */ static volatile int cpu_now_booting __cpuinitdata; -static int parisc_max_cpus __cpuinitdata = 1; +static int parisc_max_cpus = 1; static DEFINE_PER_CPU(spinlock_t, ipi_lock); @@ -328,7 +328,7 @@ void __init smp_callin(void) /* * Bring one cpu online. */ -int __cpuinit smp_boot_one_cpu(int cpuid, struct task_struct *idle) +int smp_boot_one_cpu(int cpuid, struct task_struct *idle) { const struct cpuinfo_parisc *p = &per_cpu(cpu_data, cpuid); long timeout; @@ -424,7 +424,7 @@ void smp_cpus_done(unsigned int cpu_max) } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { if (cpu != 0 && cpu < parisc_max_cpus) smp_boot_one_cpu(cpu, tidle); diff --git a/arch/powerpc/include/asm/rtas.h b/arch/powerpc/include/asm/rtas.h index c5d5cb36f6c66..50611b6438a86 100644 --- a/arch/powerpc/include/asm/rtas.h +++ b/arch/powerpc/include/asm/rtas.h @@ -351,8 +351,8 @@ static inline u32 rtas_config_addr(int busno, int devfn, int reg) (devfn << 8) | (reg & 0xff); } -extern void __cpuinit rtas_give_timebase(void); -extern void __cpuinit rtas_take_timebase(void); +extern void rtas_give_timebase(void); +extern void rtas_take_timebase(void); #ifdef CONFIG_PPC_RTAS static inline int page_is_rtas_user_buf(unsigned long pfn) diff --git a/arch/powerpc/include/asm/vdso.h b/arch/powerpc/include/asm/vdso.h index 50f261bc3e959..0d9cecddf8a4f 100644 --- a/arch/powerpc/include/asm/vdso.h +++ b/arch/powerpc/include/asm/vdso.h @@ -22,7 +22,7 @@ extern unsigned long vdso64_rt_sigtramp; extern unsigned long vdso32_sigtramp; extern unsigned long vdso32_rt_sigtramp; -int __cpuinit vdso_getcpu_init(void); +int vdso_getcpu_init(void); #else /* __ASSEMBLY__ */ diff --git a/arch/powerpc/kernel/cacheinfo.c b/arch/powerpc/kernel/cacheinfo.c index b4437e8a7a8f6..03553af1334b1 100644 --- a/arch/powerpc/kernel/cacheinfo.c +++ b/arch/powerpc/kernel/cacheinfo.c @@ -131,7 +131,7 @@ static const char *cache_type_string(const struct cache *cache) return cache_type_info[cache->type].name; } -static void __cpuinit cache_init(struct cache *cache, int type, int level, struct device_node *ofnode) +static void cache_init(struct cache *cache, int type, int level, struct device_node *ofnode) { cache->type = type; cache->level = level; @@ -140,7 +140,7 @@ static void __cpuinit cache_init(struct cache *cache, int type, int level, struc list_add(&cache->list, &cache_list); } -static struct cache *__cpuinit new_cache(int type, int level, struct device_node *ofnode) +static struct cache *new_cache(int type, int level, struct device_node *ofnode) { struct cache *cache; @@ -324,7 +324,7 @@ static bool cache_node_is_unified(const struct device_node *np) return of_get_property(np, "cache-unified", NULL); } -static struct cache *__cpuinit cache_do_one_devnode_unified(struct device_node *node, int level) +static struct cache *cache_do_one_devnode_unified(struct device_node *node, int level) { struct cache *cache; @@ -335,7 +335,7 @@ static struct cache *__cpuinit cache_do_one_devnode_unified(struct device_node * return cache; } -static struct cache *__cpuinit cache_do_one_devnode_split(struct device_node *node, int level) +static struct cache *cache_do_one_devnode_split(struct device_node *node, int level) { struct cache *dcache, *icache; @@ -357,7 +357,7 @@ static struct cache *__cpuinit cache_do_one_devnode_split(struct device_node *no return NULL; } -static struct cache *__cpuinit cache_do_one_devnode(struct device_node *node, int level) +static struct cache *cache_do_one_devnode(struct device_node *node, int level) { struct cache *cache; @@ -369,7 +369,7 @@ static struct cache *__cpuinit cache_do_one_devnode(struct device_node *node, in return cache; } -static struct cache *__cpuinit cache_lookup_or_instantiate(struct device_node *node, int level) +static struct cache *cache_lookup_or_instantiate(struct device_node *node, int level) { struct cache *cache; @@ -385,7 +385,7 @@ static struct cache *__cpuinit cache_lookup_or_instantiate(struct device_node *n return cache; } -static void __cpuinit link_cache_lists(struct cache *smaller, struct cache *bigger) +static void link_cache_lists(struct cache *smaller, struct cache *bigger) { while (smaller->next_local) { if (smaller->next_local == bigger) @@ -396,13 +396,13 @@ static void __cpuinit link_cache_lists(struct cache *smaller, struct cache *bigg smaller->next_local = bigger; } -static void __cpuinit do_subsidiary_caches_debugcheck(struct cache *cache) +static void do_subsidiary_caches_debugcheck(struct cache *cache) { WARN_ON_ONCE(cache->level != 1); WARN_ON_ONCE(strcmp(cache->ofnode->type, "cpu")); } -static void __cpuinit do_subsidiary_caches(struct cache *cache) +static void do_subsidiary_caches(struct cache *cache) { struct device_node *subcache_node; int level = cache->level; @@ -423,7 +423,7 @@ static void __cpuinit do_subsidiary_caches(struct cache *cache) } } -static struct cache *__cpuinit cache_chain_instantiate(unsigned int cpu_id) +static struct cache *cache_chain_instantiate(unsigned int cpu_id) { struct device_node *cpu_node; struct cache *cpu_cache = NULL; @@ -448,7 +448,7 @@ static struct cache *__cpuinit cache_chain_instantiate(unsigned int cpu_id) return cpu_cache; } -static struct cache_dir *__cpuinit cacheinfo_create_cache_dir(unsigned int cpu_id) +static struct cache_dir *cacheinfo_create_cache_dir(unsigned int cpu_id) { struct cache_dir *cache_dir; struct device *dev; @@ -653,7 +653,7 @@ static struct kobj_type cache_index_type = { .default_attrs = cache_index_default_attrs, }; -static void __cpuinit cacheinfo_create_index_opt_attrs(struct cache_index_dir *dir) +static void cacheinfo_create_index_opt_attrs(struct cache_index_dir *dir) { const char *cache_name; const char *cache_type; @@ -696,7 +696,7 @@ static void __cpuinit cacheinfo_create_index_opt_attrs(struct cache_index_dir *d kfree(buf); } -static void __cpuinit cacheinfo_create_index_dir(struct cache *cache, int index, struct cache_dir *cache_dir) +static void cacheinfo_create_index_dir(struct cache *cache, int index, struct cache_dir *cache_dir) { struct cache_index_dir *index_dir; int rc; @@ -722,7 +722,7 @@ static void __cpuinit cacheinfo_create_index_dir(struct cache *cache, int index, kfree(index_dir); } -static void __cpuinit cacheinfo_sysfs_populate(unsigned int cpu_id, struct cache *cache_list) +static void cacheinfo_sysfs_populate(unsigned int cpu_id, struct cache *cache_list) { struct cache_dir *cache_dir; struct cache *cache; @@ -740,7 +740,7 @@ static void __cpuinit cacheinfo_sysfs_populate(unsigned int cpu_id, struct cache } } -void __cpuinit cacheinfo_cpu_online(unsigned int cpu_id) +void cacheinfo_cpu_online(unsigned int cpu_id) { struct cache *cache; diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c index 19f49f5b535f1..6e87f4ec2ce74 100644 --- a/arch/powerpc/kernel/rtas.c +++ b/arch/powerpc/kernel/rtas.c @@ -1192,7 +1192,7 @@ int __init early_init_dt_scan_rtas(unsigned long node, static arch_spinlock_t timebase_lock; static u64 timebase = 0; -void __cpuinit rtas_give_timebase(void) +void rtas_give_timebase(void) { unsigned long flags; @@ -1209,7 +1209,7 @@ void __cpuinit rtas_give_timebase(void) local_irq_restore(flags); } -void __cpuinit rtas_take_timebase(void) +void rtas_take_timebase(void) { while (!timebase) barrier(); diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index c5c6407795491..6465fd9b6ce93 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -480,7 +480,7 @@ static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle) secondary_ti = current_set[cpu] = ti; } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { int rc, c; @@ -610,7 +610,7 @@ static struct device_node *cpu_to_l2cache(int cpu) } /* Activate a secondary processor. */ -__cpuinit void start_secondary(void *unused) +void start_secondary(void *unused) { unsigned int cpu = smp_processor_id(); struct device_node *l2_cache; diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c index a15fd1a0690c7..b4e667663d9bf 100644 --- a/arch/powerpc/kernel/sysfs.c +++ b/arch/powerpc/kernel/sysfs.c @@ -352,7 +352,7 @@ static struct device_attribute pa6t_attrs[] = { #endif /* HAS_PPC_PMC_PA6T */ #endif /* HAS_PPC_PMC_CLASSIC */ -static void __cpuinit register_cpu_online(unsigned int cpu) +static void register_cpu_online(unsigned int cpu) { struct cpu *c = &per_cpu(cpu_devices, cpu); struct device *s = &c->dev; @@ -516,7 +516,7 @@ ssize_t arch_cpu_release(const char *buf, size_t count) #endif /* CONFIG_HOTPLUG_CPU */ -static int __cpuinit sysfs_cpu_notify(struct notifier_block *self, +static int sysfs_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned int)(long)hcpu; @@ -536,7 +536,7 @@ static int __cpuinit sysfs_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata sysfs_cpu_nb = { +static struct notifier_block sysfs_cpu_nb = { .notifier_call = sysfs_cpu_notify, }; diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 57fd5c1e8e897..116b384aa7aa3 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -631,7 +631,7 @@ static int __init get_freq(char *name, int cells, unsigned long *val) return found; } -/* should become __cpuinit when secondary_cpu_time_init also is */ +/* should become when secondary_cpu_time_init also is */ void start_cpu_decrementer(void) { #if defined(CONFIG_BOOKE) || defined(CONFIG_40x) diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c index d4f463ac65b1e..1d9c92621b36f 100644 --- a/arch/powerpc/kernel/vdso.c +++ b/arch/powerpc/kernel/vdso.c @@ -711,7 +711,7 @@ static void __init vdso_setup_syscall_map(void) } #ifdef CONFIG_PPC64 -int __cpuinit vdso_getcpu_init(void) +int vdso_getcpu_init(void) { unsigned long cpu, node, val; diff --git a/arch/powerpc/mm/44x_mmu.c b/arch/powerpc/mm/44x_mmu.c index 2c9441ee6bb85..82b1ff759e26d 100644 --- a/arch/powerpc/mm/44x_mmu.c +++ b/arch/powerpc/mm/44x_mmu.c @@ -41,7 +41,7 @@ int icache_44x_need_flush; unsigned long tlb_47x_boltmap[1024/8]; -static void __cpuinit ppc44x_update_tlb_hwater(void) +static void ppc44x_update_tlb_hwater(void) { extern unsigned int tlb_44x_patch_hwater_D[]; extern unsigned int tlb_44x_patch_hwater_I[]; @@ -134,7 +134,7 @@ static void __init ppc47x_update_boltmap(void) /* * "Pins" a 256MB TLB entry in AS0 for kernel lowmem for 47x type MMU */ -static void __cpuinit ppc47x_pin_tlb(unsigned int virt, unsigned int phys) +static void ppc47x_pin_tlb(unsigned int virt, unsigned int phys) { unsigned int rA; int bolted; @@ -229,7 +229,7 @@ void setup_initial_memory_limit(phys_addr_t first_memblock_base, } #ifdef CONFIG_SMP -void __cpuinit mmu_init_secondary(int cpu) +void mmu_init_secondary(int cpu) { unsigned long addr; unsigned long memstart = memstart_addr & ~(PPC_PIN_SIZE - 1); diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index e303a6d74e3a7..4481172664dab 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -807,7 +807,7 @@ void __init early_init_mmu(void) } #ifdef CONFIG_SMP -void __cpuinit early_init_mmu_secondary(void) +void early_init_mmu_secondary(void) { /* Initialize hash table for that CPU */ if (!firmware_has_feature(FW_FEATURE_LPAR)) diff --git a/arch/powerpc/mm/mmu_context_nohash.c b/arch/powerpc/mm/mmu_context_nohash.c index e779642c25e5e..6a7238d68b2f6 100644 --- a/arch/powerpc/mm/mmu_context_nohash.c +++ b/arch/powerpc/mm/mmu_context_nohash.c @@ -329,7 +329,7 @@ void destroy_context(struct mm_struct *mm) #ifdef CONFIG_SMP -static int __cpuinit mmu_context_cpu_notify(struct notifier_block *self, +static int mmu_context_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned int)(long)hcpu; @@ -363,7 +363,7 @@ static int __cpuinit mmu_context_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata mmu_context_cpu_nb = { +static struct notifier_block mmu_context_cpu_nb = { .notifier_call = mmu_context_cpu_notify, }; diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c index 08c6f3185d45b..9ddf2e065ab0e 100644 --- a/arch/powerpc/mm/numa.c +++ b/arch/powerpc/mm/numa.c @@ -532,7 +532,7 @@ static int of_drconf_to_nid_single(struct of_drconf_cell *drmem, * Figure out to which domain a cpu belongs and stick it there. * Return the id of the domain used. */ -static int __cpuinit numa_setup_cpu(unsigned long lcpu) +static int numa_setup_cpu(unsigned long lcpu) { int nid; struct device_node *cpu; @@ -567,7 +567,7 @@ static int __cpuinit numa_setup_cpu(unsigned long lcpu) return nid; } -static int __cpuinit cpu_numa_callback(struct notifier_block *nfb, +static int cpu_numa_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { @@ -948,7 +948,7 @@ static void __init *careful_zallocation(int nid, unsigned long size, return ret; } -static struct notifier_block __cpuinitdata ppc64_numa_nb = { +static struct notifier_block ppc64_numa_nb = { .notifier_call = cpu_numa_callback, .priority = 1 /* Must run before sched domains notifier. */ }; diff --git a/arch/powerpc/mm/tlb_nohash.c b/arch/powerpc/mm/tlb_nohash.c index 6888cad5103de..41cd68dee6816 100644 --- a/arch/powerpc/mm/tlb_nohash.c +++ b/arch/powerpc/mm/tlb_nohash.c @@ -648,7 +648,7 @@ void __init early_init_mmu(void) __early_init_mmu(1); } -void __cpuinit early_init_mmu_secondary(void) +void early_init_mmu_secondary(void) { __early_init_mmu(0); } diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index b63dc809596d5..e4a99f12e9937 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -1825,7 +1825,7 @@ static void power_pmu_setup(int cpu) cpuhw->mmcr[0] = MMCR0_FC; } -static int __cpuinit +static int power_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (long)hcpu; @@ -1842,7 +1842,7 @@ power_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu return NOTIFY_OK; } -int __cpuinit register_power_pmu(struct power_pmu *pmu) +int register_power_pmu(struct power_pmu *pmu) { if (ppmu) return -EBUSY; /* something's already registered */ diff --git a/arch/powerpc/platforms/44x/currituck.c b/arch/powerpc/platforms/44x/currituck.c index ecd3890c40d72..9cef9d381b05c 100644 --- a/arch/powerpc/platforms/44x/currituck.c +++ b/arch/powerpc/platforms/44x/currituck.c @@ -91,12 +91,12 @@ static void __init ppc47x_init_irq(void) } #ifdef CONFIG_SMP -static void __cpuinit smp_ppc47x_setup_cpu(int cpu) +static void smp_ppc47x_setup_cpu(int cpu) { mpic_setup_this_cpu(); } -static int __cpuinit smp_ppc47x_kick_cpu(int cpu) +static int smp_ppc47x_kick_cpu(int cpu) { struct device_node *cpunode = of_get_cpu_node(cpu, NULL); const u64 *spin_table_addr_prop; diff --git a/arch/powerpc/platforms/44x/iss4xx.c b/arch/powerpc/platforms/44x/iss4xx.c index a28a8629727ee..4241bc825800d 100644 --- a/arch/powerpc/platforms/44x/iss4xx.c +++ b/arch/powerpc/platforms/44x/iss4xx.c @@ -81,12 +81,12 @@ static void __init iss4xx_init_irq(void) } #ifdef CONFIG_SMP -static void __cpuinit smp_iss4xx_setup_cpu(int cpu) +static void smp_iss4xx_setup_cpu(int cpu) { mpic_setup_this_cpu(); } -static int __cpuinit smp_iss4xx_kick_cpu(int cpu) +static int smp_iss4xx_kick_cpu(int cpu) { struct device_node *cpunode = of_get_cpu_node(cpu, NULL); const u64 *spin_table_addr_prop; diff --git a/arch/powerpc/platforms/85xx/smp.c b/arch/powerpc/platforms/85xx/smp.c index 6a1759939c6ba..5ced4f5bb2b2e 100644 --- a/arch/powerpc/platforms/85xx/smp.c +++ b/arch/powerpc/platforms/85xx/smp.c @@ -99,7 +99,7 @@ static void mpc85xx_take_timebase(void) } #ifdef CONFIG_HOTPLUG_CPU -static void __cpuinit smp_85xx_mach_cpu_die(void) +static void smp_85xx_mach_cpu_die(void) { unsigned int cpu = smp_processor_id(); u32 tmp; @@ -141,7 +141,7 @@ static inline u32 read_spin_table_addr_l(void *spin_table) return in_be32(&((struct epapr_spin_table *)spin_table)->addr_l); } -static int __cpuinit smp_85xx_kick_cpu(int nr) +static int smp_85xx_kick_cpu(int nr) { unsigned long flags; const u64 *cpu_rel_addr; @@ -362,7 +362,7 @@ static void mpc85xx_smp_machine_kexec(struct kimage *image) } #endif /* CONFIG_KEXEC */ -static void __cpuinit smp_85xx_setup_cpu(int cpu_nr) +static void smp_85xx_setup_cpu(int cpu_nr) { if (smp_85xx_ops.probe == smp_mpic_probe) mpic_setup_this_cpu(); diff --git a/arch/powerpc/platforms/powermac/smp.c b/arch/powerpc/platforms/powermac/smp.c index bdb738a69e418..49c9f9501c213 100644 --- a/arch/powerpc/platforms/powermac/smp.c +++ b/arch/powerpc/platforms/powermac/smp.c @@ -885,7 +885,7 @@ static int smp_core99_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata smp_core99_cpu_nb = { +static struct notifier_block smp_core99_cpu_nb = { .notifier_call = smp_core99_cpu_notify, }; #endif /* CONFIG_HOTPLUG_CPU */ diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c index 88c9459c3e071..c22b2b3d9d84f 100644 --- a/arch/powerpc/platforms/powernv/smp.c +++ b/arch/powerpc/platforms/powernv/smp.c @@ -40,7 +40,7 @@ #define DBG(fmt...) #endif -static void __cpuinit pnv_smp_setup_cpu(int cpu) +static void pnv_smp_setup_cpu(int cpu) { if (cpu != boot_cpuid) xics_setup_cpu(); diff --git a/arch/s390/kernel/cache.c b/arch/s390/kernel/cache.c index 64b24650e4f8c..ae32bfca2f3d1 100644 --- a/arch/s390/kernel/cache.c +++ b/arch/s390/kernel/cache.c @@ -173,7 +173,7 @@ static void __init cache_build_info(void) } } -static struct cache_dir *__cpuinit cache_create_cache_dir(int cpu) +static struct cache_dir *cache_create_cache_dir(int cpu) { struct cache_dir *cache_dir; struct kobject *kobj = NULL; @@ -289,7 +289,7 @@ static struct kobj_type cache_index_type = { .default_attrs = cache_index_default_attrs, }; -static int __cpuinit cache_create_index_dir(struct cache_dir *cache_dir, +static int cache_create_index_dir(struct cache_dir *cache_dir, struct cache *cache, int index, int cpu) { @@ -313,7 +313,7 @@ static int __cpuinit cache_create_index_dir(struct cache_dir *cache_dir, return rc; } -static int __cpuinit cache_add_cpu(int cpu) +static int cache_add_cpu(int cpu) { struct cache_dir *cache_dir; struct cache *cache; @@ -335,7 +335,7 @@ static int __cpuinit cache_add_cpu(int cpu) return 0; } -static void __cpuinit cache_remove_cpu(int cpu) +static void cache_remove_cpu(int cpu) { struct cache_index_dir *index, *next; struct cache_dir *cache_dir; @@ -354,7 +354,7 @@ static void __cpuinit cache_remove_cpu(int cpu) cache_dir_cpu[cpu] = NULL; } -static int __cpuinit cache_hotplug(struct notifier_block *nfb, +static int cache_hotplug(struct notifier_block *nfb, unsigned long action, void *hcpu) { int cpu = (long)hcpu; diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 390d9ae57bb2d..e7518bb6c3130 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -639,7 +639,7 @@ static struct pmu cpumf_pmu = { .cancel_txn = cpumf_pmu_cancel_txn, }; -static int __cpuinit cpumf_pmu_notifier(struct notifier_block *self, +static int cpumf_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (long) hcpu; diff --git a/arch/s390/kernel/processor.c b/arch/s390/kernel/processor.c index 753c41d0ffd3a..24612029f4502 100644 --- a/arch/s390/kernel/processor.c +++ b/arch/s390/kernel/processor.c @@ -21,7 +21,7 @@ static DEFINE_PER_CPU(struct cpuid, cpu_id); /* * cpu_init - initializes state that is per-CPU. */ -void __cpuinit cpu_init(void) +void cpu_init(void) { struct s390_idle_data *idle = &__get_cpu_var(s390_idle); struct cpuid *id = &__get_cpu_var(cpu_id); diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 14647fe09d0cd..4266590ff7012 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -166,7 +166,7 @@ static void pcpu_ec_call(struct pcpu *pcpu, int ec_bit) pcpu_sigp_retry(pcpu, order, 0); } -static int __cpuinit pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) +static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) { struct _lowcore *lc; @@ -619,9 +619,9 @@ static struct sclp_cpu_info *smp_get_cpu_info(void) return info; } -static int __cpuinit smp_add_present_cpu(int cpu); +static int smp_add_present_cpu(int cpu); -static int __cpuinit __smp_rescan_cpus(struct sclp_cpu_info *info, +static int __smp_rescan_cpus(struct sclp_cpu_info *info, int sysfs_add) { struct pcpu *pcpu; @@ -688,7 +688,7 @@ static void __init smp_detect_cpus(void) /* * Activate a secondary processor. */ -static void __cpuinit smp_start_secondary(void *cpuvoid) +static void smp_start_secondary(void *cpuvoid) { S390_lowcore.last_update_clock = get_tod_clock(); S390_lowcore.restart_stack = (unsigned long) restart_stack; @@ -711,7 +711,7 @@ static void __cpuinit smp_start_secondary(void *cpuvoid) } /* Upping and downing of CPUs */ -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { struct pcpu *pcpu; int rc; @@ -967,7 +967,7 @@ static struct attribute_group cpu_online_attr_group = { .attrs = cpu_online_attrs, }; -static int __cpuinit smp_cpu_notify(struct notifier_block *self, +static int smp_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned int)(long)hcpu; @@ -986,7 +986,7 @@ static int __cpuinit smp_cpu_notify(struct notifier_block *self, return notifier_from_errno(err); } -static int __cpuinit smp_add_present_cpu(int cpu) +static int smp_add_present_cpu(int cpu) { struct cpu *c = &pcpu_devices[cpu].cpu; struct device *s = &c->dev; diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c index 62f89d98e8807..811f542b8ed4a 100644 --- a/arch/s390/kernel/sysinfo.c +++ b/arch/s390/kernel/sysinfo.c @@ -418,7 +418,7 @@ void s390_adjust_jiffies(void) /* * calibrate the delay loop */ -void __cpuinit calibrate_delay(void) +void calibrate_delay(void) { s390_adjust_jiffies(); /* Print the good old Bogomips line .. */ diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index 737d50caa4fea..a37d56a338395 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -371,13 +371,13 @@ EXPORT_SYMBOL(del_virt_timer); /* * Start the virtual CPU timer on the current CPU. */ -void __cpuinit init_cpu_vtimer(void) +void init_cpu_vtimer(void) { /* set initial cpu timer */ set_vtimer(VTIMER_MAX_SLICE); } -static int __cpuinit s390_nohz_notify(struct notifier_block *self, +static int s390_nohz_notify(struct notifier_block *self, unsigned long action, void *hcpu) { struct s390_idle_data *idle; diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index d214321db727f..245cfa9b3a670 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -647,7 +647,7 @@ static void pfault_interrupt(struct ext_code ext_code, put_task_struct(tsk); } -static int __cpuinit pfault_cpu_notify(struct notifier_block *self, +static int pfault_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { struct thread_struct *thread, *next; diff --git a/arch/score/mm/tlb-score.c b/arch/score/mm/tlb-score.c index 6fdb100244c81..004073717de03 100644 --- a/arch/score/mm/tlb-score.c +++ b/arch/score/mm/tlb-score.c @@ -240,7 +240,7 @@ void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte) local_irq_restore(flags); } -void __cpuinit tlb_init(void) +void tlb_init(void) { tlblock_set(0); local_flush_tlb_all(); diff --git a/arch/sh/kernel/cpu/init.c b/arch/sh/kernel/cpu/init.c index 61a07dafcd46c..0a2ef460e19d7 100644 --- a/arch/sh/kernel/cpu/init.c +++ b/arch/sh/kernel/cpu/init.c @@ -43,9 +43,9 @@ * peripherals (nofpu, nodsp, and so forth). */ #define onchip_setup(x) \ -static int x##_disabled __cpuinitdata = !cpu_has_##x; \ +static int x##_disabled = !cpu_has_##x; \ \ -static int __cpuinit x##_setup(char *opts) \ +static int x##_setup(char *opts) \ { \ x##_disabled = 1; \ return 1; \ @@ -59,7 +59,7 @@ onchip_setup(dsp); #define CPUOPM 0xff2f0000 #define CPUOPM_RABD (1 << 5) -static void __cpuinit speculative_execution_init(void) +static void speculative_execution_init(void) { /* Clear RABD */ __raw_writel(__raw_readl(CPUOPM) & ~CPUOPM_RABD, CPUOPM); @@ -78,7 +78,7 @@ static void __cpuinit speculative_execution_init(void) #define EXPMASK_BRDSSLP (1 << 1) #define EXPMASK_MMCAW (1 << 4) -static void __cpuinit expmask_init(void) +static void expmask_init(void) { unsigned long expmask = __raw_readl(EXPMASK); @@ -217,7 +217,7 @@ static void detect_cache_shape(void) l2_cache_shape = -1; /* No S-cache */ } -static void __cpuinit fpu_init(void) +static void fpu_init(void) { /* Disable the FPU */ if (fpu_disabled && (current_cpu_data.flags & CPU_HAS_FPU)) { @@ -230,7 +230,7 @@ static void __cpuinit fpu_init(void) } #ifdef CONFIG_SH_DSP -static void __cpuinit release_dsp(void) +static void release_dsp(void) { unsigned long sr; @@ -244,7 +244,7 @@ static void __cpuinit release_dsp(void) ); } -static void __cpuinit dsp_init(void) +static void dsp_init(void) { unsigned long sr; @@ -276,7 +276,7 @@ static void __cpuinit dsp_init(void) release_dsp(); } #else -static inline void __cpuinit dsp_init(void) { } +static inline void dsp_init(void) { } #endif /* CONFIG_SH_DSP */ /** @@ -295,7 +295,7 @@ static inline void __cpuinit dsp_init(void) { } * Each processor family is still responsible for doing its own probing * and cache configuration in cpu_probe(). */ -asmlinkage void __cpuinit cpu_init(void) +asmlinkage void cpu_init(void) { current_thread_info()->cpu = hard_smp_processor_id(); diff --git a/arch/sh/kernel/cpu/sh2/probe.c b/arch/sh/kernel/cpu/sh2/probe.c index bab8e75958aee..6c687ae812ef6 100644 --- a/arch/sh/kernel/cpu/sh2/probe.c +++ b/arch/sh/kernel/cpu/sh2/probe.c @@ -13,7 +13,7 @@ #include #include -void __cpuinit cpu_probe(void) +void cpu_probe(void) { #if defined(CONFIG_CPU_SUBTYPE_SH7619) boot_cpu_data.type = CPU_SH7619; diff --git a/arch/sh/kernel/cpu/sh2a/probe.c b/arch/sh/kernel/cpu/sh2a/probe.c index 5170b6aa4129e..3f87971082f11 100644 --- a/arch/sh/kernel/cpu/sh2a/probe.c +++ b/arch/sh/kernel/cpu/sh2a/probe.c @@ -13,7 +13,7 @@ #include #include -void __cpuinit cpu_probe(void) +void cpu_probe(void) { boot_cpu_data.family = CPU_FAMILY_SH2A; diff --git a/arch/sh/kernel/cpu/sh3/probe.c b/arch/sh/kernel/cpu/sh3/probe.c index bf23c322e164b..426e1e1dcedc7 100644 --- a/arch/sh/kernel/cpu/sh3/probe.c +++ b/arch/sh/kernel/cpu/sh3/probe.c @@ -16,7 +16,7 @@ #include #include -void __cpuinit cpu_probe(void) +void cpu_probe(void) { unsigned long addr0, addr1, data0, data1, data2, data3; diff --git a/arch/sh/kernel/cpu/sh4/probe.c b/arch/sh/kernel/cpu/sh4/probe.c index 0fbbd50bc8ad3..a521bcf506956 100644 --- a/arch/sh/kernel/cpu/sh4/probe.c +++ b/arch/sh/kernel/cpu/sh4/probe.c @@ -15,7 +15,7 @@ #include #include -void __cpuinit cpu_probe(void) +void cpu_probe(void) { unsigned long pvr, prr, cvr; unsigned long size; diff --git a/arch/sh/kernel/cpu/sh4a/smp-shx3.c b/arch/sh/kernel/cpu/sh4a/smp-shx3.c index 03f2b55757cfc..4a298808789c4 100644 --- a/arch/sh/kernel/cpu/sh4a/smp-shx3.c +++ b/arch/sh/kernel/cpu/sh4a/smp-shx3.c @@ -124,7 +124,7 @@ static void shx3_update_boot_vector(unsigned int cpu) __raw_writel(STBCR_RESET, STBCR_REG(cpu)); } -static int __cpuinit +static int shx3_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned int)hcpu; @@ -143,11 +143,11 @@ shx3_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) return NOTIFY_OK; } -static struct notifier_block __cpuinitdata shx3_cpu_notifier = { +static struct notifier_block shx3_cpu_notifier = { .notifier_call = shx3_cpu_callback, }; -static int __cpuinit register_shx3_cpu_notifier(void) +static int register_shx3_cpu_notifier(void) { register_hotcpu_notifier(&shx3_cpu_notifier); return 0; diff --git a/arch/sh/kernel/cpu/sh5/probe.c b/arch/sh/kernel/cpu/sh5/probe.c index 9e882409e4e98..eca427c2f2f3d 100644 --- a/arch/sh/kernel/cpu/sh5/probe.c +++ b/arch/sh/kernel/cpu/sh5/probe.c @@ -17,7 +17,7 @@ #include #include -void __cpuinit cpu_probe(void) +void cpu_probe(void) { unsigned long long cir; diff --git a/arch/sh/kernel/perf_event.c b/arch/sh/kernel/perf_event.c index 068b8a2759b52..b9cefebda55c0 100644 --- a/arch/sh/kernel/perf_event.c +++ b/arch/sh/kernel/perf_event.c @@ -367,7 +367,7 @@ static void sh_pmu_setup(int cpu) memset(cpuhw, 0, sizeof(struct cpu_hw_events)); } -static int __cpuinit +static int sh_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (long)hcpu; @@ -384,7 +384,7 @@ sh_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) return NOTIFY_OK; } -int __cpuinit register_sh_pmu(struct sh_pmu *_pmu) +int register_sh_pmu(struct sh_pmu *_pmu) { if (sh_pmu) return -EBUSY; diff --git a/arch/sh/kernel/process.c b/arch/sh/kernel/process.c index 055d91b70305f..53bc6c4c84ecd 100644 --- a/arch/sh/kernel/process.c +++ b/arch/sh/kernel/process.c @@ -65,7 +65,7 @@ void arch_task_cache_init(void) # define HAVE_SOFTFP 0 #endif -void __cpuinit init_thread_xstate(void) +void init_thread_xstate(void) { if (boot_cpu_data.flags & CPU_HAS_FPU) xstate_size = sizeof(struct sh_fpu_hard_struct); diff --git a/arch/sh/kernel/setup.c b/arch/sh/kernel/setup.c index ebe7a7d97215e..1cf90e947dbf1 100644 --- a/arch/sh/kernel/setup.c +++ b/arch/sh/kernel/setup.c @@ -172,7 +172,7 @@ void __init check_for_initrd(void) #endif } -void __cpuinit calibrate_delay(void) +void calibrate_delay(void) { struct clk *clk = clk_get(NULL, "cpu_clk"); diff --git a/arch/sh/kernel/smp.c b/arch/sh/kernel/smp.c index 45696451f0ea3..86a7936a980b6 100644 --- a/arch/sh/kernel/smp.c +++ b/arch/sh/kernel/smp.c @@ -37,7 +37,7 @@ struct plat_smp_ops *mp_ops = NULL; /* State of each CPU */ DEFINE_PER_CPU(int, cpu_state) = { 0 }; -void __cpuinit register_smp_ops(struct plat_smp_ops *ops) +void register_smp_ops(struct plat_smp_ops *ops) { if (mp_ops) printk(KERN_WARNING "Overriding previously set SMP ops\n"); @@ -45,7 +45,7 @@ void __cpuinit register_smp_ops(struct plat_smp_ops *ops) mp_ops = ops; } -static inline void __cpuinit smp_store_cpu_info(unsigned int cpu) +static inline void smp_store_cpu_info(unsigned int cpu) { struct sh_cpuinfo *c = cpu_data + cpu; @@ -174,7 +174,7 @@ void native_play_dead(void) } #endif -asmlinkage void __cpuinit start_secondary(void) +asmlinkage void start_secondary(void) { unsigned int cpu = smp_processor_id(); struct mm_struct *mm = &init_mm; @@ -215,7 +215,7 @@ extern struct { void *thread_info; } stack_start; -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tsk) +int __cpu_up(unsigned int cpu, struct task_struct *tsk) { unsigned long timeout; diff --git a/arch/sh/kernel/traps_32.c b/arch/sh/kernel/traps_32.c index 5f513a64dedfc..68e99f09171dd 100644 --- a/arch/sh/kernel/traps_32.c +++ b/arch/sh/kernel/traps_32.c @@ -741,7 +741,7 @@ asmlinkage void do_exception_error(unsigned long r4, unsigned long r5, die_if_kernel("exception", regs, ex); } -void __cpuinit per_cpu_trap_init(void) +void per_cpu_trap_init(void) { extern void *vbr_base; diff --git a/arch/sh/kernel/traps_64.c b/arch/sh/kernel/traps_64.c index f87d20da17916..112ea11c030d6 100644 --- a/arch/sh/kernel/traps_64.c +++ b/arch/sh/kernel/traps_64.c @@ -810,7 +810,7 @@ asmlinkage void do_debug_interrupt(unsigned long code, struct pt_regs *regs) poke_real_address_q(DM_EXP_CAUSE_PHY, 0x0); } -void __cpuinit per_cpu_trap_init(void) +void per_cpu_trap_init(void) { /* Nothing to do for now, VBR initialization later. */ } diff --git a/arch/sh/mm/tlb-sh5.c b/arch/sh/mm/tlb-sh5.c index ff1c40a31cbc0..e4bb2a8e0a69d 100644 --- a/arch/sh/mm/tlb-sh5.c +++ b/arch/sh/mm/tlb-sh5.c @@ -17,7 +17,7 @@ /** * sh64_tlb_init - Perform initial setup for the DTLB and ITLB. */ -int __cpuinit sh64_tlb_init(void) +int sh64_tlb_init(void) { /* Assign some sane DTLB defaults */ cpu_data->dtlb.entries = 64; diff --git a/arch/sparc/kernel/ds.c b/arch/sparc/kernel/ds.c index 252f8768aac69..3f7d4b372f3a6 100644 --- a/arch/sparc/kernel/ds.c +++ b/arch/sparc/kernel/ds.c @@ -528,7 +528,7 @@ static void dr_cpu_mark(struct ds_data *resp, int cpu, int ncpus, } } -static int __cpuinit dr_cpu_configure(struct ds_info *dp, +static int dr_cpu_configure(struct ds_info *dp, struct ds_cap_state *cp, u64 req_num, cpumask_t *mask) @@ -627,7 +627,7 @@ static int dr_cpu_unconfigure(struct ds_info *dp, return 0; } -static void __cpuinit dr_cpu_data(struct ds_info *dp, +static void dr_cpu_data(struct ds_info *dp, struct ds_cap_state *cp, void *buf, int len) { diff --git a/arch/sparc/kernel/entry.h b/arch/sparc/kernel/entry.h index cc3c5cb47cdaa..9c179fbfb219c 100644 --- a/arch/sparc/kernel/entry.h +++ b/arch/sparc/kernel/entry.h @@ -250,7 +250,7 @@ extern struct ino_bucket *ivector_table; extern unsigned long ivector_table_pa; extern void init_irqwork_curcpu(void); -extern void __cpuinit sun4v_register_mondo_queues(int this_cpu); +extern void sun4v_register_mondo_queues(int this_cpu); #endif /* CONFIG_SPARC32 */ #endif /* _ENTRY_H */ diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c index 9bcbbe2c4e7e1..d16d16520678c 100644 --- a/arch/sparc/kernel/irq_64.c +++ b/arch/sparc/kernel/irq_64.c @@ -835,7 +835,7 @@ void notrace init_irqwork_curcpu(void) * Therefore you cannot make any OBP calls, not even prom_printf, * from these two routines. */ -static void __cpuinit notrace register_one_mondo(unsigned long paddr, unsigned long type, unsigned long qmask) +static void notrace register_one_mondo(unsigned long paddr, unsigned long type, unsigned long qmask) { unsigned long num_entries = (qmask + 1) / 64; unsigned long status; @@ -848,7 +848,7 @@ static void __cpuinit notrace register_one_mondo(unsigned long paddr, unsigned l } } -void __cpuinit notrace sun4v_register_mondo_queues(int this_cpu) +void notrace sun4v_register_mondo_queues(int this_cpu) { struct trap_per_cpu *tb = &trap_block[this_cpu]; diff --git a/arch/sparc/kernel/leon_smp.c b/arch/sparc/kernel/leon_smp.c index 6cfc1b09ec25a..b5945694bef63 100644 --- a/arch/sparc/kernel/leon_smp.c +++ b/arch/sparc/kernel/leon_smp.c @@ -54,7 +54,7 @@ extern ctxd_t *srmmu_ctx_table_phys; static int smp_processors_ready; extern volatile unsigned long cpu_callin_map[NR_CPUS]; extern cpumask_t smp_commenced_mask; -void __cpuinit leon_configure_cache_smp(void); +void leon_configure_cache_smp(void); static void leon_ipi_init(void); /* IRQ number of LEON IPIs */ @@ -69,12 +69,12 @@ static inline unsigned long do_swap(volatile unsigned long *ptr, return val; } -void __cpuinit leon_cpu_pre_starting(void *arg) +void leon_cpu_pre_starting(void *arg) { leon_configure_cache_smp(); } -void __cpuinit leon_cpu_pre_online(void *arg) +void leon_cpu_pre_online(void *arg) { int cpuid = hard_smp_processor_id(); @@ -106,7 +106,7 @@ void __cpuinit leon_cpu_pre_online(void *arg) extern struct linux_prom_registers smp_penguin_ctable; -void __cpuinit leon_configure_cache_smp(void) +void leon_configure_cache_smp(void) { unsigned long cfg = sparc_leon3_get_dcachecfg(); int me = smp_processor_id(); @@ -186,7 +186,7 @@ void __init leon_boot_cpus(void) } -int __cpuinit leon_boot_one_cpu(int i, struct task_struct *idle) +int leon_boot_one_cpu(int i, struct task_struct *idle) { int timeout; diff --git a/arch/sparc/kernel/mdesc.c b/arch/sparc/kernel/mdesc.c index 831c001604e8e..3961054c595c4 100644 --- a/arch/sparc/kernel/mdesc.c +++ b/arch/sparc/kernel/mdesc.c @@ -571,7 +571,7 @@ static void __init report_platform_properties(void) mdesc_release(hp); } -static void __cpuinit fill_in_one_cache(cpuinfo_sparc *c, +static void fill_in_one_cache(cpuinfo_sparc *c, struct mdesc_handle *hp, u64 mp) { @@ -616,7 +616,7 @@ static void __cpuinit fill_in_one_cache(cpuinfo_sparc *c, } } -static void __cpuinit mark_core_ids(struct mdesc_handle *hp, u64 mp, int core_id) +static void mark_core_ids(struct mdesc_handle *hp, u64 mp, int core_id) { u64 a; @@ -649,7 +649,7 @@ static void __cpuinit mark_core_ids(struct mdesc_handle *hp, u64 mp, int core_id } } -static void __cpuinit set_core_ids(struct mdesc_handle *hp) +static void set_core_ids(struct mdesc_handle *hp) { int idx; u64 mp; @@ -674,7 +674,7 @@ static void __cpuinit set_core_ids(struct mdesc_handle *hp) } } -static void __cpuinit mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id) +static void mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id) { u64 a; @@ -693,7 +693,7 @@ static void __cpuinit mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id } } -static void __cpuinit __set_proc_ids(struct mdesc_handle *hp, const char *exec_unit_name) +static void __set_proc_ids(struct mdesc_handle *hp, const char *exec_unit_name) { int idx; u64 mp; @@ -714,13 +714,13 @@ static void __cpuinit __set_proc_ids(struct mdesc_handle *hp, const char *exec_u } } -static void __cpuinit set_proc_ids(struct mdesc_handle *hp) +static void set_proc_ids(struct mdesc_handle *hp) { __set_proc_ids(hp, "exec_unit"); __set_proc_ids(hp, "exec-unit"); } -static void __cpuinit get_one_mondo_bits(const u64 *p, unsigned int *mask, +static void get_one_mondo_bits(const u64 *p, unsigned int *mask, unsigned long def, unsigned long max) { u64 val; @@ -742,7 +742,7 @@ static void __cpuinit get_one_mondo_bits(const u64 *p, unsigned int *mask, *mask = ((1U << def) * 64U) - 1U; } -static void __cpuinit get_mondo_data(struct mdesc_handle *hp, u64 mp, +static void get_mondo_data(struct mdesc_handle *hp, u64 mp, struct trap_per_cpu *tb) { static int printed; @@ -769,7 +769,7 @@ static void __cpuinit get_mondo_data(struct mdesc_handle *hp, u64 mp, } } -static void * __cpuinit mdesc_iterate_over_cpus(void *(*func)(struct mdesc_handle *, u64, int, void *), void *arg, cpumask_t *mask) +static void * mdesc_iterate_over_cpus(void *(*func)(struct mdesc_handle *, u64, int, void *), void *arg, cpumask_t *mask) { struct mdesc_handle *hp = mdesc_grab(); void *ret = NULL; @@ -799,7 +799,7 @@ static void * __cpuinit mdesc_iterate_over_cpus(void *(*func)(struct mdesc_handl return ret; } -static void * __cpuinit record_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, void *arg) +static void * record_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, void *arg) { ncpus_probed++; #ifdef CONFIG_SMP @@ -808,7 +808,7 @@ static void * __cpuinit record_one_cpu(struct mdesc_handle *hp, u64 mp, int cpui return NULL; } -void __cpuinit mdesc_populate_present_mask(cpumask_t *mask) +void mdesc_populate_present_mask(cpumask_t *mask) { if (tlb_type != hypervisor) return; @@ -841,7 +841,7 @@ void __init mdesc_get_page_sizes(cpumask_t *mask, unsigned long *pgsz_mask) mdesc_iterate_over_cpus(check_one_pgsz, pgsz_mask, mask); } -static void * __cpuinit fill_in_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, void *arg) +static void * fill_in_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, void *arg) { const u64 *cfreq = mdesc_get_property(hp, mp, "clock-frequency", NULL); struct trap_per_cpu *tb; @@ -890,7 +890,7 @@ static void * __cpuinit fill_in_one_cpu(struct mdesc_handle *hp, u64 mp, int cpu return NULL; } -void __cpuinit mdesc_fill_in_cpu_data(cpumask_t *mask) +void mdesc_fill_in_cpu_data(cpumask_t *mask) { struct mdesc_handle *hp; diff --git a/arch/sparc/kernel/smp_32.c b/arch/sparc/kernel/smp_32.c index e3f2b81c23f14..a102bfba6ea86 100644 --- a/arch/sparc/kernel/smp_32.c +++ b/arch/sparc/kernel/smp_32.c @@ -39,7 +39,7 @@ #include "kernel.h" #include "irq.h" -volatile unsigned long cpu_callin_map[NR_CPUS] __cpuinitdata = {0,}; +volatile unsigned long cpu_callin_map[NR_CPUS] = {0,}; cpumask_t smp_commenced_mask = CPU_MASK_NONE; @@ -53,7 +53,7 @@ const struct sparc32_ipi_ops *sparc32_ipi_ops; * instruction which is much better... */ -void __cpuinit smp_store_cpu_info(int id) +void smp_store_cpu_info(int id) { int cpu_node; int mid; @@ -120,7 +120,7 @@ void cpu_panic(void) panic("SMP bolixed\n"); } -struct linux_prom_registers smp_penguin_ctable __cpuinitdata = { 0 }; +struct linux_prom_registers smp_penguin_ctable = { 0 }; void smp_send_reschedule(int cpu) { @@ -259,10 +259,10 @@ void __init smp_prepare_boot_cpu(void) set_cpu_possible(cpuid, true); } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { - extern int __cpuinit smp4m_boot_one_cpu(int, struct task_struct *); - extern int __cpuinit smp4d_boot_one_cpu(int, struct task_struct *); + extern int smp4m_boot_one_cpu(int, struct task_struct *); + extern int smp4d_boot_one_cpu(int, struct task_struct *); int ret=0; switch(sparc_cpu_model) { @@ -297,7 +297,7 @@ int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) return ret; } -void __cpuinit arch_cpu_pre_starting(void *arg) +void arch_cpu_pre_starting(void *arg) { local_ops->cache_all(); local_ops->tlb_all(); @@ -317,7 +317,7 @@ void __cpuinit arch_cpu_pre_starting(void *arg) } } -void __cpuinit arch_cpu_pre_online(void *arg) +void arch_cpu_pre_online(void *arg) { unsigned int cpuid = hard_smp_processor_id(); @@ -344,7 +344,7 @@ void __cpuinit arch_cpu_pre_online(void *arg) } } -void __cpuinit sparc_start_secondary(void *arg) +void sparc_start_secondary(void *arg) { unsigned int cpu; @@ -375,7 +375,7 @@ void __cpuinit sparc_start_secondary(void *arg) BUG(); } -void __cpuinit smp_callin(void) +void smp_callin(void) { sparc_start_secondary(NULL); } diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index 173964d5e948a..7c3d82107a61c 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c @@ -87,7 +87,7 @@ extern void setup_sparc64_timer(void); static volatile unsigned long callin_flag = 0; -void __cpuinit smp_callin(void) +void smp_callin(void) { int cpuid = hard_smp_processor_id(); @@ -281,7 +281,7 @@ static unsigned long kimage_addr_to_ra(void *p) return kern_base + (val - KERNBASE); } -static void __cpuinit ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread_reg, void **descrp) +static void ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread_reg, void **descrp) { extern unsigned long sparc64_ttable_tl0; extern unsigned long kern_locked_tte_data; @@ -342,7 +342,7 @@ extern unsigned long sparc64_cpu_startup; */ static struct thread_info *cpu_new_thread = NULL; -static int __cpuinit smp_boot_one_cpu(unsigned int cpu, struct task_struct *idle) +static int smp_boot_one_cpu(unsigned int cpu, struct task_struct *idle) { unsigned long entry = (unsigned long)(&sparc64_cpu_startup); @@ -1270,7 +1270,7 @@ void smp_fill_in_sib_core_maps(void) } } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { int ret = smp_boot_one_cpu(cpu, tidle); diff --git a/arch/sparc/kernel/sun4d_smp.c b/arch/sparc/kernel/sun4d_smp.c index c9eb82f23d928..d5c319553fd08 100644 --- a/arch/sparc/kernel/sun4d_smp.c +++ b/arch/sparc/kernel/sun4d_smp.c @@ -50,7 +50,7 @@ static inline void show_leds(int cpuid) "i" (ASI_M_CTL)); } -void __cpuinit sun4d_cpu_pre_starting(void *arg) +void sun4d_cpu_pre_starting(void *arg) { int cpuid = hard_smp_processor_id(); @@ -62,7 +62,7 @@ void __cpuinit sun4d_cpu_pre_starting(void *arg) cc_set_imsk((cc_get_imsk() & ~0x8000) | 0x4000); } -void __cpuinit sun4d_cpu_pre_online(void *arg) +void sun4d_cpu_pre_online(void *arg) { unsigned long flags; int cpuid; @@ -118,7 +118,7 @@ void __init smp4d_boot_cpus(void) local_ops->cache_all(); } -int __cpuinit smp4d_boot_one_cpu(int i, struct task_struct *idle) +int smp4d_boot_one_cpu(int i, struct task_struct *idle) { unsigned long *entry = &sun4d_cpu_startup; int timeout; diff --git a/arch/sparc/kernel/sun4m_smp.c b/arch/sparc/kernel/sun4m_smp.c index 8a65f158153dd..d3408e72d20c0 100644 --- a/arch/sparc/kernel/sun4m_smp.c +++ b/arch/sparc/kernel/sun4m_smp.c @@ -34,11 +34,11 @@ swap_ulong(volatile unsigned long *ptr, unsigned long val) return val; } -void __cpuinit sun4m_cpu_pre_starting(void *arg) +void sun4m_cpu_pre_starting(void *arg) { } -void __cpuinit sun4m_cpu_pre_online(void *arg) +void sun4m_cpu_pre_online(void *arg) { int cpuid = hard_smp_processor_id(); @@ -75,7 +75,7 @@ void __init smp4m_boot_cpus(void) local_ops->cache_all(); } -int __cpuinit smp4m_boot_one_cpu(int i, struct task_struct *idle) +int smp4m_boot_one_cpu(int i, struct task_struct *idle) { unsigned long *entry = &sun4m_cpu_startup; int timeout; diff --git a/arch/sparc/kernel/sysfs.c b/arch/sparc/kernel/sysfs.c index 654e8aad3bbef..c21c673e5f7cd 100644 --- a/arch/sparc/kernel/sysfs.c +++ b/arch/sparc/kernel/sysfs.c @@ -246,7 +246,7 @@ static void unregister_cpu_online(unsigned int cpu) } #endif -static int __cpuinit sysfs_cpu_notify(struct notifier_block *self, +static int sysfs_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned int)(long)hcpu; @@ -266,7 +266,7 @@ static int __cpuinit sysfs_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata sysfs_cpu_nb = { +static struct notifier_block sysfs_cpu_nb = { .notifier_call = sysfs_cpu_notify, }; diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index 272ebe7d380d4..04541b0cd9833 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c @@ -1698,7 +1698,7 @@ static void __init sun4v_ktsb_init(void) #endif } -void __cpuinit sun4v_ktsb_register(void) +void sun4v_ktsb_register(void) { unsigned long pa, ret; diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index f58cb540ff948..1e0e7d2837da1 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -860,7 +860,7 @@ static void __init map_kernel(void) } } -void (*poke_srmmu)(void) __cpuinitdata = NULL; +void (*poke_srmmu)(void) = NULL; extern unsigned long bootmem_init(unsigned long *pages_avail); @@ -1058,7 +1058,7 @@ static void __init init_vac_layout(void) (int)vac_cache_size, (int)vac_line_size); } -static void __cpuinit poke_hypersparc(void) +static void poke_hypersparc(void) { volatile unsigned long clear; unsigned long mreg = srmmu_get_mmureg(); @@ -1110,7 +1110,7 @@ static void __init init_hypersparc(void) hypersparc_setup_blockops(); } -static void __cpuinit poke_swift(void) +static void poke_swift(void) { unsigned long mreg; @@ -1290,7 +1290,7 @@ static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long } -static void __cpuinit poke_turbosparc(void) +static void poke_turbosparc(void) { unsigned long mreg = srmmu_get_mmureg(); unsigned long ccreg; @@ -1353,7 +1353,7 @@ static void __init init_turbosparc(void) poke_srmmu = poke_turbosparc; } -static void __cpuinit poke_tsunami(void) +static void poke_tsunami(void) { unsigned long mreg = srmmu_get_mmureg(); @@ -1394,7 +1394,7 @@ static void __init init_tsunami(void) tsunami_setup_blockops(); } -static void __cpuinit poke_viking(void) +static void poke_viking(void) { unsigned long mreg = srmmu_get_mmureg(); static int smp_catch; diff --git a/arch/tile/kernel/irq.c b/arch/tile/kernel/irq.c index 02e6280650125..3ccf2cd7182eb 100644 --- a/arch/tile/kernel/irq.c +++ b/arch/tile/kernel/irq.c @@ -220,7 +220,7 @@ void __init init_IRQ(void) ipi_init(); } -void __cpuinit setup_irq_regs(void) +void setup_irq_regs(void) { /* Enable interrupt delivery. */ unmask_irqs(~0UL); diff --git a/arch/tile/kernel/messaging.c b/arch/tile/kernel/messaging.c index 0858ee6b520f6..00331af9525dc 100644 --- a/arch/tile/kernel/messaging.c +++ b/arch/tile/kernel/messaging.c @@ -25,7 +25,7 @@ /* All messages are stored here */ static DEFINE_PER_CPU(HV_MsgState, msg_state); -void __cpuinit init_messaging(void) +void init_messaging(void) { /* Allocate storage for messages in kernel space */ HV_MsgState *state = &__get_cpu_var(msg_state); diff --git a/arch/tile/kernel/setup.c b/arch/tile/kernel/setup.c index 5ebe936762016..0d7c1f30349ea 100644 --- a/arch/tile/kernel/setup.c +++ b/arch/tile/kernel/setup.c @@ -58,8 +58,8 @@ struct pglist_data node_data[MAX_NUMNODES] __read_mostly; EXPORT_SYMBOL(node_data); /* Information on the NUMA nodes that we compute early */ -unsigned long __cpuinitdata node_start_pfn[MAX_NUMNODES]; -unsigned long __cpuinitdata node_end_pfn[MAX_NUMNODES]; +unsigned long node_start_pfn[MAX_NUMNODES]; +unsigned long node_end_pfn[MAX_NUMNODES]; unsigned long __initdata node_memmap_pfn[MAX_NUMNODES]; unsigned long __initdata node_percpu_pfn[MAX_NUMNODES]; unsigned long __initdata node_free_pfn[MAX_NUMNODES]; @@ -84,7 +84,7 @@ unsigned long __initdata boot_pc = (unsigned long)start_kernel; #ifdef CONFIG_HIGHMEM /* Page frame index of end of lowmem on each controller. */ -unsigned long __cpuinitdata node_lowmem_end_pfn[MAX_NUMNODES]; +unsigned long node_lowmem_end_pfn[MAX_NUMNODES]; /* Number of pages that can be mapped into lowmem. */ static unsigned long __initdata mappable_physpages; @@ -290,7 +290,7 @@ static void *__init setup_pa_va_mapping(void) * This is up to 4 mappings for lowmem, one mapping per memory * controller, plus one for our text segment. */ -static void __cpuinit store_permanent_mappings(void) +static void store_permanent_mappings(void) { int i; @@ -935,7 +935,7 @@ subsys_initcall(topology_init); * So the values we set up here in the hypervisor may be overridden on * the boot cpu as arguments are parsed. */ -static __cpuinit void init_super_pages(void) +static void init_super_pages(void) { #ifdef CONFIG_HUGETLB_SUPER_PAGES int i; @@ -950,7 +950,7 @@ static __cpuinit void init_super_pages(void) * * Called from setup_arch() on the boot cpu, or online_secondary(). */ -void __cpuinit setup_cpu(int boot) +void setup_cpu(int boot) { /* The boot cpu sets up its permanent mappings much earlier. */ if (!boot) diff --git a/arch/tile/kernel/smpboot.c b/arch/tile/kernel/smpboot.c index 44bab29bf2f3a..c4d09761a3838 100644 --- a/arch/tile/kernel/smpboot.c +++ b/arch/tile/kernel/smpboot.c @@ -140,7 +140,7 @@ static struct cpumask cpu_started __cpuinitdata; * to this path without knowing what you're doing, since SMP booting * is pretty fragile. */ -static void __cpuinit start_secondary(void) +static void start_secondary(void) { int cpuid = smp_processor_id(); @@ -183,7 +183,7 @@ static void __cpuinit start_secondary(void) /* * Bring a secondary processor online. */ -void __cpuinit online_secondary(void) +void online_secondary(void) { /* * low-memory mappings have been cleared, flush them from @@ -210,7 +210,7 @@ void __cpuinit online_secondary(void) cpu_startup_entry(CPUHP_ONLINE); } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { /* Wait 5s total for all CPUs for them to come online */ static int timeout; diff --git a/arch/tile/kernel/time.c b/arch/tile/kernel/time.c index 9df6d0d6d187e..251c05d50060d 100644 --- a/arch/tile/kernel/time.c +++ b/arch/tile/kernel/time.c @@ -159,7 +159,7 @@ static DEFINE_PER_CPU(struct clock_event_device, tile_timer) = { .set_mode = tile_timer_set_mode, }; -void __cpuinit setup_tile_timer(void) +void setup_tile_timer(void) { struct clock_event_device *evt = &__get_cpu_var(tile_timer); diff --git a/arch/x86/include/asm/cpu.h b/arch/x86/include/asm/cpu.h index 5f9a1243190e8..d2b12988d2ed1 100644 --- a/arch/x86/include/asm/cpu.h +++ b/arch/x86/include/asm/cpu.h @@ -28,7 +28,7 @@ struct x86_cpu { #ifdef CONFIG_HOTPLUG_CPU extern int arch_register_cpu(int num); extern void arch_unregister_cpu(int); -extern void __cpuinit start_cpu0(void); +extern void start_cpu0(void); #ifdef CONFIG_DEBUG_HOTPLUG_CPU0 extern int _debug_hotplug_cpu(int cpu, int action); #endif diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h index 6bc3985ee473d..f98bd66253182 100644 --- a/arch/x86/include/asm/microcode.h +++ b/arch/x86/include/asm/microcode.h @@ -60,11 +60,11 @@ static inline void __exit exit_amd_microcode(void) {} #ifdef CONFIG_MICROCODE_EARLY #define MAX_UCODE_COUNT 128 extern void __init load_ucode_bsp(void); -extern void __cpuinit load_ucode_ap(void); +extern void load_ucode_ap(void); extern int __init save_microcode_in_initrd(void); #else static inline void __init load_ucode_bsp(void) {} -static inline void __cpuinit load_ucode_ap(void) {} +static inline void load_ucode_ap(void) {} static inline int __init save_microcode_in_initrd(void) { return 0; diff --git a/arch/x86/include/asm/microcode_intel.h b/arch/x86/include/asm/microcode_intel.h index 5356f927d411c..62c1f79625777 100644 --- a/arch/x86/include/asm/microcode_intel.h +++ b/arch/x86/include/asm/microcode_intel.h @@ -65,11 +65,11 @@ update_match_revision(struct microcode_header_intel *mc_header, int rev); #ifdef CONFIG_MICROCODE_INTEL_EARLY extern void __init load_ucode_intel_bsp(void); -extern void __cpuinit load_ucode_intel_ap(void); +extern void load_ucode_intel_ap(void); extern void show_ucode_info_early(void); #else static inline __init void load_ucode_intel_bsp(void) {} -static inline __cpuinit void load_ucode_intel_ap(void) {} +static inline void load_ucode_intel_ap(void) {} static inline void show_ucode_info_early(void) {} #endif diff --git a/arch/x86/include/asm/mmconfig.h b/arch/x86/include/asm/mmconfig.h index 9b119da1d105f..04a3fed22cfe0 100644 --- a/arch/x86/include/asm/mmconfig.h +++ b/arch/x86/include/asm/mmconfig.h @@ -2,8 +2,8 @@ #define _ASM_X86_MMCONFIG_H #ifdef CONFIG_PCI_MMCONFIG -extern void __cpuinit fam10h_check_enable_mmcfg(void); -extern void __cpuinit check_enable_amd_mmconf_dmi(void); +extern void fam10h_check_enable_mmcfg(void); +extern void check_enable_amd_mmconf_dmi(void); #else static inline void fam10h_check_enable_mmcfg(void) { } static inline void check_enable_amd_mmconf_dmi(void) { } diff --git a/arch/x86/include/asm/mpspec.h b/arch/x86/include/asm/mpspec.h index 3e2f42a4b872a..626cf70082d7c 100644 --- a/arch/x86/include/asm/mpspec.h +++ b/arch/x86/include/asm/mpspec.h @@ -94,7 +94,7 @@ static inline void early_reserve_e820_mpc_new(void) { } #define default_get_smp_config x86_init_uint_noop #endif -void __cpuinit generic_processor_info(int apicid, int version); +void generic_processor_info(int apicid, int version); #ifdef CONFIG_ACPI extern void mp_register_ioapic(int id, u32 address, u32 gsi_base); extern void mp_override_legacy_irq(u8 bus_irq, u8 polarity, u8 trigger, diff --git a/arch/x86/include/asm/numa.h b/arch/x86/include/asm/numa.h index 1b99ee5c9f00d..4064acae625d9 100644 --- a/arch/x86/include/asm/numa.h +++ b/arch/x86/include/asm/numa.h @@ -39,7 +39,7 @@ static inline void set_apicid_to_node(int apicid, s16 node) __apicid_to_node[apicid] = node; } -extern int __cpuinit numa_cpu_node(int cpu); +extern int numa_cpu_node(int cpu); #else /* CONFIG_NUMA */ static inline void set_apicid_to_node(int apicid, s16 node) @@ -60,8 +60,8 @@ static inline int numa_cpu_node(int cpu) extern void numa_set_node(int cpu, int node); extern void numa_clear_node(int cpu); extern void __init init_cpu_to_node(void); -extern void __cpuinit numa_add_cpu(int cpu); -extern void __cpuinit numa_remove_cpu(int cpu); +extern void numa_add_cpu(int cpu); +extern void numa_remove_cpu(int cpu); #else /* CONFIG_NUMA */ static inline void numa_set_node(int cpu, int node) { } static inline void numa_clear_node(int cpu) { } diff --git a/arch/x86/include/asm/prom.h b/arch/x86/include/asm/prom.h index 60bef663609a4..bade6ac3b14fc 100644 --- a/arch/x86/include/asm/prom.h +++ b/arch/x86/include/asm/prom.h @@ -27,7 +27,7 @@ extern int of_ioapic; extern u64 initial_dtb; extern void add_dtb(u64 data); extern void x86_add_irq_domains(void); -void __cpuinit x86_of_pci_init(void); +void x86_of_pci_init(void); void x86_dtb_init(void); #else static inline void add_dtb(u64 data) { } diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index b073aaea747c2..1e2b33829cb2b 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -179,7 +179,7 @@ static inline int wbinvd_on_all_cpus(void) } #endif /* CONFIG_SMP */ -extern unsigned disabled_cpus __cpuinitdata; +extern unsigned disabled_cpusdata; #ifdef CONFIG_X86_32_SMP /* diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 230c8ea878e5f..6c96b60fe3f39 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -194,7 +194,7 @@ static int __init acpi_parse_madt(struct acpi_table_header *table) return 0; } -static void __cpuinit acpi_register_lapic(int id, u8 enabled) +static void acpi_register_lapic(int id, u8 enabled) { unsigned int ver = 0; @@ -600,7 +600,7 @@ void __init acpi_set_irq_model_ioapic(void) #ifdef CONFIG_ACPI_HOTPLUG_CPU #include -static void __cpuinit acpi_map_cpu2node(acpi_handle handle, int cpu, int physid) +static void acpi_map_cpu2node(acpi_handle handle, int cpu, int physid) { #ifdef CONFIG_ACPI_NUMA int nid; @@ -613,7 +613,7 @@ static void __cpuinit acpi_map_cpu2node(acpi_handle handle, int cpu, int physid) #endif } -static int __cpuinit _acpi_map_lsapic(acpi_handle handle, int *pcpu) +static int _acpi_map_lsapic(acpi_handle handle, int *pcpu) { struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; union acpi_object *obj; diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 3cd8bfc3c4b69..17a5c0863a45f 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -550,7 +550,7 @@ static DEFINE_PER_CPU(struct clock_event_device, lapic_events); * Setup the local APIC timer for this CPU. Copy the initialized values * of the boot CPU and register the clock event in the framework. */ -static void __cpuinit setup_APIC_timer(void) +static void setup_APIC_timer(void) { struct clock_event_device *levt = &__get_cpu_var(lapic_events); @@ -872,7 +872,7 @@ void __init setup_boot_APIC_clock(void) setup_APIC_timer(); } -void __cpuinit setup_secondary_APIC_clock(void) +void setup_secondary_APIC_clock(void) { setup_APIC_timer(); } @@ -1217,7 +1217,7 @@ void __init init_bsp_APIC(void) apic_write(APIC_LVT1, value); } -static void __cpuinit lapic_setup_esr(void) +static void lapic_setup_esr(void) { unsigned int oldvalue, value, maxlvt; @@ -1264,7 +1264,7 @@ static void __cpuinit lapic_setup_esr(void) * Used to setup local APIC while initializing BSP or bringin up APs. * Always called with preemption disabled. */ -void __cpuinit setup_local_APIC(void) +void setup_local_APIC(void) { int cpu = smp_processor_id(); unsigned int value, queued; @@ -1459,7 +1459,7 @@ void __cpuinit setup_local_APIC(void) #endif } -void __cpuinit end_local_APIC_setup(void) +void end_local_APIC_setup(void) { lapic_setup_esr(); @@ -2072,7 +2072,7 @@ void disconnect_bsp_APIC(int virt_wire_setup) apic_write(APIC_LVT1, value); } -void __cpuinit generic_processor_info(int apicid, int version) +void generic_processor_info(int apicid, int version) { int cpu, max = nr_cpu_ids; bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid, @@ -2342,7 +2342,7 @@ static struct syscore_ops lapic_syscore_ops = { .suspend = lapic_suspend, }; -static void __cpuinit apic_pm_activate(void) +static void apic_pm_activate(void) { apic_pm_state.active = 1; } @@ -2367,7 +2367,7 @@ static void apic_pm_activate(void) { } #ifdef CONFIG_X86_64 -static int __cpuinit apic_cluster_num(void) +static int apic_cluster_num(void) { int i, clusters, zeros; unsigned id; @@ -2412,10 +2412,10 @@ static int __cpuinit apic_cluster_num(void) return clusters; } -static int __cpuinitdata multi_checked; -static int __cpuinitdata multi; +static int multi_checked; +static int multi; -static int __cpuinit set_multi(const struct dmi_system_id *d) +static int set_multi(const struct dmi_system_id *d) { if (multi) return 0; @@ -2424,7 +2424,7 @@ static int __cpuinit set_multi(const struct dmi_system_id *d) return 0; } -static const __cpuinitconst struct dmi_system_id multi_dmi_table[] = { +static const struct dmi_system_id multi_dmi_table[] = { { .callback = set_multi, .ident = "IBM System Summit2", @@ -2436,7 +2436,7 @@ static const __cpuinitconst struct dmi_system_id multi_dmi_table[] = { {} }; -static void __cpuinit dmi_check_multi(void) +static void dmi_check_multi(void) { if (multi_checked) return; @@ -2453,7 +2453,7 @@ static void __cpuinit dmi_check_multi(void) * multi-chassis. * Use DMI to check them */ -__cpuinit int apic_is_clustered_box(void) +int apic_is_clustered_box(void) { dmi_check_multi(); if (multi) diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index 9a9110918ca71..3e67f9e3d7ef1 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -74,7 +74,7 @@ static int numachip_phys_pkg_id(int initial_apic_id, int index_msb) return initial_apic_id >> index_msb; } -static int __cpuinit numachip_wakeup_secondary(int phys_apicid, unsigned long start_rip) +static int numachip_wakeup_secondary(int phys_apicid, unsigned long start_rip) { union numachip_csr_g3_ext_irq_gen int_gen; diff --git a/arch/x86/kernel/apic/es7000_32.c b/arch/x86/kernel/apic/es7000_32.c index 0874799a98c6f..c55224731b2d7 100644 --- a/arch/x86/kernel/apic/es7000_32.c +++ b/arch/x86/kernel/apic/es7000_32.c @@ -130,7 +130,7 @@ int es7000_plat; */ -static int __cpuinit wakeup_secondary_cpu_via_mip(int cpu, unsigned long eip) +static int wakeup_secondary_cpu_via_mip(int cpu, unsigned long eip) { unsigned long vect = 0, psaival = 0; diff --git a/arch/x86/kernel/apic/numaq_32.c b/arch/x86/kernel/apic/numaq_32.c index d661ee95cabfa..1e42e8f305ee2 100644 --- a/arch/x86/kernel/apic/numaq_32.c +++ b/arch/x86/kernel/apic/numaq_32.c @@ -105,7 +105,7 @@ static void __init smp_dump_qct(void) } } -void __cpuinit numaq_tsc_disable(void) +void numaq_tsc_disable(void) { if (!found_numaq) return; diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index c88baa4ff0e56..140e29db478d7 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -148,7 +148,7 @@ static void init_x2apic_ldr(void) /* * At CPU state changes, update the x2apic cluster sibling info. */ -static int __cpuinit +static int update_clusterinfo(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int this_cpu = (unsigned long)hcpu; diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index b32dbb411a9ae..578d821d75f25 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -194,7 +194,7 @@ EXPORT_SYMBOL_GPL(uv_possible_blades); unsigned long sn_rtc_cycles_per_second; EXPORT_SYMBOL(sn_rtc_cycles_per_second); -static int __cpuinit uv_wakeup_secondary(int phys_apicid, unsigned long start_rip) +static int uv_wakeup_secondary(int phys_apicid, unsigned long start_rip) { #ifdef CONFIG_SMP unsigned long val; @@ -401,7 +401,7 @@ static struct apic __refdata apic_x2apic_uv_x = { .safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle, }; -static __cpuinit void set_x2apic_extra_bits(int pnode) +static void set_x2apic_extra_bits(int pnode) { __this_cpu_write(x2apic_extra_bits, pnode << uvh_apicid.s.pnode_shift); } @@ -683,7 +683,7 @@ static void uv_heartbeat(unsigned long ignored) mod_timer_pinned(timer, jiffies + SCIR_CPU_HB_INTERVAL); } -static void __cpuinit uv_heartbeat_enable(int cpu) +static void uv_heartbeat_enable(int cpu) { while (!uv_cpu_hub_info(cpu)->scir.enabled) { struct timer_list *timer = &uv_cpu_hub_info(cpu)->scir.timer; @@ -700,7 +700,7 @@ static void __cpuinit uv_heartbeat_enable(int cpu) } #ifdef CONFIG_HOTPLUG_CPU -static void __cpuinit uv_heartbeat_disable(int cpu) +static void uv_heartbeat_disable(int cpu) { if (uv_cpu_hub_info(cpu)->scir.enabled) { uv_cpu_hub_info(cpu)->scir.enabled = 0; @@ -712,7 +712,7 @@ static void __cpuinit uv_heartbeat_disable(int cpu) /* * cpu hotplug notifier */ -static __cpuinit int uv_scir_cpu_notify(struct notifier_block *self, +static int uv_scir_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { long cpu = (long)hcpu; @@ -783,7 +783,7 @@ int uv_set_vga_state(struct pci_dev *pdev, bool decode, * Called on each cpu to initialize the per_cpu UV data area. * FIXME: hotplug not supported yet */ -void __cpuinit uv_cpu_init(void) +void uv_cpu_init(void) { /* CPU 0 initilization will be done via uv_system_init. */ if (!uv_blade_info) diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index ae177a014180b..bb4f57d5b7fd8 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -69,7 +69,7 @@ static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val) extern void vide(void); __asm__(".align 4\nvide: ret"); -static void __cpuinit init_amd_k5(struct cpuinfo_x86 *c) +static void init_amd_k5(struct cpuinfo_x86 *c) { /* * General Systems BIOSen alias the cpu frequency registers @@ -87,7 +87,7 @@ static void __cpuinit init_amd_k5(struct cpuinfo_x86 *c) } -static void __cpuinit init_amd_k6(struct cpuinfo_x86 *c) +static void init_amd_k6(struct cpuinfo_x86 *c) { u32 l, h; int mbytes = num_physpages >> (20-PAGE_SHIFT); @@ -179,7 +179,7 @@ static void __cpuinit init_amd_k6(struct cpuinfo_x86 *c) } } -static void __cpuinit amd_k7_smp_check(struct cpuinfo_x86 *c) +static void amd_k7_smp_check(struct cpuinfo_x86 *c) { /* calling is from identify_secondary_cpu() ? */ if (!c->cpu_index) @@ -222,7 +222,7 @@ static void __cpuinit amd_k7_smp_check(struct cpuinfo_x86 *c) add_taint(TAINT_UNSAFE_SMP, LOCKDEP_NOW_UNRELIABLE); } -static void __cpuinit init_amd_k7(struct cpuinfo_x86 *c) +static void init_amd_k7(struct cpuinfo_x86 *c) { u32 l, h; @@ -267,7 +267,7 @@ static void __cpuinit init_amd_k7(struct cpuinfo_x86 *c) * To workaround broken NUMA config. Read the comment in * srat_detect_node(). */ -static int __cpuinit nearby_node(int apicid) +static int nearby_node(int apicid) { int i, node; @@ -292,7 +292,7 @@ static int __cpuinit nearby_node(int apicid) * (2) AMD processors supporting compute units */ #ifdef CONFIG_X86_HT -static void __cpuinit amd_get_topology(struct cpuinfo_x86 *c) +static void amd_get_topology(struct cpuinfo_x86 *c) { u32 nodes, cores_per_cu = 1; u8 node_id; @@ -342,7 +342,7 @@ static void __cpuinit amd_get_topology(struct cpuinfo_x86 *c) * On a AMD dual core setup the lower bits of the APIC id distingush the cores. * Assumes number of cores is a power of two. */ -static void __cpuinit amd_detect_cmp(struct cpuinfo_x86 *c) +static void amd_detect_cmp(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_HT unsigned bits; @@ -369,7 +369,7 @@ u16 amd_get_nb_id(int cpu) } EXPORT_SYMBOL_GPL(amd_get_nb_id); -static void __cpuinit srat_detect_node(struct cpuinfo_x86 *c) +static void srat_detect_node(struct cpuinfo_x86 *c) { #ifdef CONFIG_NUMA int cpu = smp_processor_id(); @@ -421,7 +421,7 @@ static void __cpuinit srat_detect_node(struct cpuinfo_x86 *c) #endif } -static void __cpuinit early_init_amd_mc(struct cpuinfo_x86 *c) +static void early_init_amd_mc(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_HT unsigned bits, ecx; @@ -447,7 +447,7 @@ static void __cpuinit early_init_amd_mc(struct cpuinfo_x86 *c) #endif } -static void __cpuinit bsp_init_amd(struct cpuinfo_x86 *c) +static void bsp_init_amd(struct cpuinfo_x86 *c) { if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) { @@ -475,7 +475,7 @@ static void __cpuinit bsp_init_amd(struct cpuinfo_x86 *c) } } -static void __cpuinit early_init_amd(struct cpuinfo_x86 *c) +static void early_init_amd(struct cpuinfo_x86 *c) { early_init_amd_mc(c); @@ -524,7 +524,7 @@ static const int amd_erratum_383[]; static const int amd_erratum_400[]; static bool cpu_has_amd_erratum(const int *erratum); -static void __cpuinit init_amd(struct cpuinfo_x86 *c) +static void init_amd(struct cpuinfo_x86 *c) { u32 dummy; unsigned long long value; @@ -750,7 +750,7 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c) } #ifdef CONFIG_X86_32 -static unsigned int __cpuinit amd_size_cache(struct cpuinfo_x86 *c, +static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size) { /* AMD errata T13 (order #21922) */ @@ -767,7 +767,7 @@ static unsigned int __cpuinit amd_size_cache(struct cpuinfo_x86 *c, } #endif -static void __cpuinit cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c) +static void cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c) { tlb_flushall_shift = 5; @@ -775,7 +775,7 @@ static void __cpuinit cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c) tlb_flushall_shift = 4; } -static void __cpuinit cpu_detect_tlb_amd(struct cpuinfo_x86 *c) +static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c) { u32 ebx, eax, ecx, edx; u16 mask = 0xfff; @@ -830,7 +830,7 @@ static void __cpuinit cpu_detect_tlb_amd(struct cpuinfo_x86 *c) cpu_set_tlb_flushall_shift(c); } -static const struct cpu_dev __cpuinitconst amd_cpu_dev = { +static const struct cpu_dev amd_cpu_dev = { .c_vendor = "AMD", .c_ident = { "AuthenticAMD" }, #ifdef CONFIG_X86_32 diff --git a/arch/x86/kernel/cpu/centaur.c b/arch/x86/kernel/cpu/centaur.c index 159103c0b1f4b..fbf6c3bc2400a 100644 --- a/arch/x86/kernel/cpu/centaur.c +++ b/arch/x86/kernel/cpu/centaur.c @@ -11,7 +11,7 @@ #ifdef CONFIG_X86_OOSTORE -static u32 __cpuinit power2(u32 x) +static u32 power2(u32 x) { u32 s = 1; @@ -25,7 +25,7 @@ static u32 __cpuinit power2(u32 x) /* * Set up an actual MCR */ -static void __cpuinit centaur_mcr_insert(int reg, u32 base, u32 size, int key) +static void centaur_mcr_insert(int reg, u32 base, u32 size, int key) { u32 lo, hi; @@ -42,7 +42,7 @@ static void __cpuinit centaur_mcr_insert(int reg, u32 base, u32 size, int key) * * Shortcut: We know you can't put 4Gig of RAM on a winchip */ -static u32 __cpuinit ramtop(void) +static u32 ramtop(void) { u32 clip = 0xFFFFFFFFUL; u32 top = 0; @@ -91,7 +91,7 @@ static u32 __cpuinit ramtop(void) /* * Compute a set of MCR's to give maximum coverage */ -static int __cpuinit centaur_mcr_compute(int nr, int key) +static int centaur_mcr_compute(int nr, int key) { u32 mem = ramtop(); u32 root = power2(mem); @@ -157,7 +157,7 @@ static int __cpuinit centaur_mcr_compute(int nr, int key) return ct; } -static void __cpuinit centaur_create_optimal_mcr(void) +static void centaur_create_optimal_mcr(void) { int used; int i; @@ -181,7 +181,7 @@ static void __cpuinit centaur_create_optimal_mcr(void) wrmsr(MSR_IDT_MCR0+i, 0, 0); } -static void __cpuinit winchip2_create_optimal_mcr(void) +static void winchip2_create_optimal_mcr(void) { u32 lo, hi; int used; @@ -217,7 +217,7 @@ static void __cpuinit winchip2_create_optimal_mcr(void) /* * Handle the MCR key on the Winchip 2. */ -static void __cpuinit winchip2_unprotect_mcr(void) +static void winchip2_unprotect_mcr(void) { u32 lo, hi; u32 key; @@ -229,7 +229,7 @@ static void __cpuinit winchip2_unprotect_mcr(void) wrmsr(MSR_IDT_MCR_CTRL, lo, hi); } -static void __cpuinit winchip2_protect_mcr(void) +static void winchip2_protect_mcr(void) { u32 lo, hi; @@ -247,7 +247,7 @@ static void __cpuinit winchip2_protect_mcr(void) #define RNG_ENABLED (1 << 3) #define RNG_ENABLE (1 << 6) /* MSR_VIA_RNG */ -static void __cpuinit init_c3(struct cpuinfo_x86 *c) +static void init_c3(struct cpuinfo_x86 *c) { u32 lo, hi; @@ -318,7 +318,7 @@ enum { EAMD3D = 1<<20, }; -static void __cpuinit early_init_centaur(struct cpuinfo_x86 *c) +static void early_init_centaur(struct cpuinfo_x86 *c) { switch (c->x86) { #ifdef CONFIG_X86_32 @@ -337,7 +337,7 @@ static void __cpuinit early_init_centaur(struct cpuinfo_x86 *c) #endif } -static void __cpuinit init_centaur(struct cpuinfo_x86 *c) +static void init_centaur(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_32 char *name; @@ -468,7 +468,7 @@ static void __cpuinit init_centaur(struct cpuinfo_x86 *c) #endif } -static unsigned int __cpuinit +static unsigned int centaur_size_cache(struct cpuinfo_x86 *c, unsigned int size) { #ifdef CONFIG_X86_32 @@ -488,7 +488,7 @@ centaur_size_cache(struct cpuinfo_x86 *c, unsigned int size) return size; } -static const struct cpu_dev __cpuinitconst centaur_cpu_dev = { +static const struct cpu_dev centaur_cpu_dev = { .c_vendor = "Centaur", .c_ident = { "CentaurHauls" }, .c_early_init = early_init_centaur, diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 7e09789d2cf38..6a9c29e1a8c50 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -63,7 +63,7 @@ void __init setup_cpu_local_masks(void) alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask); } -static void __cpuinit default_init(struct cpuinfo_x86 *c) +static void default_init(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_64 cpu_detect_cache_sizes(c); @@ -80,13 +80,13 @@ static void __cpuinit default_init(struct cpuinfo_x86 *c) #endif } -static const struct cpu_dev __cpuinitconst default_cpu = { +static const struct cpu_dev default_cpu = { .c_init = default_init, .c_vendor = "Unknown", .c_x86_vendor = X86_VENDOR_UNKNOWN, }; -static const struct cpu_dev *this_cpu __cpuinitdata = &default_cpu; +static const struct cpu_dev *this_cpu = &default_cpu; DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = { #ifdef CONFIG_X86_64 @@ -162,8 +162,8 @@ static int __init x86_xsaveopt_setup(char *s) __setup("noxsaveopt", x86_xsaveopt_setup); #ifdef CONFIG_X86_32 -static int cachesize_override __cpuinitdata = -1; -static int disable_x86_serial_nr __cpuinitdata = 1; +static int cachesize_override = -1; +static int disable_x86_serial_nr = 1; static int __init cachesize_setup(char *str) { @@ -217,12 +217,12 @@ static inline int flag_is_changeable_p(u32 flag) } /* Probe for the CPUID instruction */ -int __cpuinit have_cpuid_p(void) +int have_cpuid_p(void) { return flag_is_changeable_p(X86_EFLAGS_ID); } -static void __cpuinit squash_the_stupid_serial_number(struct cpuinfo_x86 *c) +static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c) { unsigned long lo, hi; @@ -312,7 +312,7 @@ cpuid_dependent_features[] = { { 0, 0 } }; -static void __cpuinit filter_cpuid_features(struct cpuinfo_x86 *c, bool warn) +static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn) { const struct cpuid_dependent_feature *df; @@ -350,7 +350,7 @@ static void __cpuinit filter_cpuid_features(struct cpuinfo_x86 *c, bool warn) */ /* Look up CPU names by table lookup. */ -static const char *__cpuinit table_lookup_model(struct cpuinfo_x86 *c) +static const char *table_lookup_model(struct cpuinfo_x86 *c) { const struct cpu_model_info *info; @@ -400,9 +400,9 @@ void switch_to_new_gdt(int cpu) load_percpu_segment(cpu); } -static const struct cpu_dev *__cpuinitdata cpu_devs[X86_VENDOR_NUM] = {}; +static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {}; -static void __cpuinit get_model_name(struct cpuinfo_x86 *c) +static void get_model_name(struct cpuinfo_x86 *c) { unsigned int *v; char *p, *q; @@ -431,7 +431,7 @@ static void __cpuinit get_model_name(struct cpuinfo_x86 *c) } } -void __cpuinit cpu_detect_cache_sizes(struct cpuinfo_x86 *c) +void cpu_detect_cache_sizes(struct cpuinfo_x86 *c) { unsigned int n, dummy, ebx, ecx, edx, l2size; @@ -485,7 +485,7 @@ u16 __read_mostly tlb_lld_4m[NR_INFO]; */ s8 __read_mostly tlb_flushall_shift = -1; -void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c) +void cpu_detect_tlb(struct cpuinfo_x86 *c) { if (this_cpu->c_detect_tlb) this_cpu->c_detect_tlb(c); @@ -499,7 +499,7 @@ void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c) tlb_flushall_shift); } -void __cpuinit detect_ht(struct cpuinfo_x86 *c) +void detect_ht(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_HT u32 eax, ebx, ecx, edx; @@ -550,7 +550,7 @@ void __cpuinit detect_ht(struct cpuinfo_x86 *c) #endif } -static void __cpuinit get_cpu_vendor(struct cpuinfo_x86 *c) +static void get_cpu_vendor(struct cpuinfo_x86 *c) { char *v = c->x86_vendor_id; int i; @@ -577,7 +577,7 @@ static void __cpuinit get_cpu_vendor(struct cpuinfo_x86 *c) this_cpu = &default_cpu; } -void __cpuinit cpu_detect(struct cpuinfo_x86 *c) +void cpu_detect(struct cpuinfo_x86 *c) { /* Get vendor name */ cpuid(0x00000000, (unsigned int *)&c->cpuid_level, @@ -607,7 +607,7 @@ void __cpuinit cpu_detect(struct cpuinfo_x86 *c) } } -void __cpuinit get_cpu_cap(struct cpuinfo_x86 *c) +void get_cpu_cap(struct cpuinfo_x86 *c) { u32 tfms, xlvl; u32 ebx; @@ -658,7 +658,7 @@ void __cpuinit get_cpu_cap(struct cpuinfo_x86 *c) init_scattered_cpuid_features(c); } -static void __cpuinit identify_cpu_without_cpuid(struct cpuinfo_x86 *c) +static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_32 int i; @@ -774,7 +774,7 @@ void __init early_cpu_init(void) * unless we can find a reliable way to detect all the broken cases. * Enable it explicitly on 64-bit for non-constant inputs of cpu_has(). */ -static void __cpuinit detect_nopl(struct cpuinfo_x86 *c) +static void detect_nopl(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_32 clear_cpu_cap(c, X86_FEATURE_NOPL); @@ -783,7 +783,7 @@ static void __cpuinit detect_nopl(struct cpuinfo_x86 *c) #endif } -static void __cpuinit generic_identify(struct cpuinfo_x86 *c) +static void generic_identify(struct cpuinfo_x86 *c) { c->extended_cpuid_level = 0; @@ -820,7 +820,7 @@ static void __cpuinit generic_identify(struct cpuinfo_x86 *c) /* * This does the hard work of actually picking apart the CPU stuff... */ -static void __cpuinit identify_cpu(struct cpuinfo_x86 *c) +static void identify_cpu(struct cpuinfo_x86 *c) { int i; @@ -965,7 +965,7 @@ void __init identify_boot_cpu(void) cpu_detect_tlb(&boot_cpu_data); } -void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c) +void identify_secondary_cpu(struct cpuinfo_x86 *c) { BUG_ON(c == &boot_cpu_data); identify_cpu(c); @@ -980,14 +980,14 @@ struct msr_range { unsigned max; }; -static const struct msr_range msr_range_array[] __cpuinitconst = { +static const struct msr_range msr_range_array[] = { { 0x00000000, 0x00000418}, { 0xc0000000, 0xc000040b}, { 0xc0010000, 0xc0010142}, { 0xc0011000, 0xc001103b}, }; -static void __cpuinit __print_cpu_msr(void) +static void __print_cpu_msr(void) { unsigned index_min, index_max; unsigned index; @@ -1027,7 +1027,7 @@ static __init int setup_noclflush(char *arg) } __setup("noclflush", setup_noclflush); -void __cpuinit print_cpu_info(struct cpuinfo_x86 *c) +void print_cpu_info(struct cpuinfo_x86 *c) { const char *vendor = NULL; @@ -1056,7 +1056,7 @@ void __cpuinit print_cpu_info(struct cpuinfo_x86 *c) print_cpu_msr(c); } -void __cpuinit print_cpu_msr(struct cpuinfo_x86 *c) +void print_cpu_msr(struct cpuinfo_x86 *c) { if (c->cpu_index < show_msr) __print_cpu_msr(); @@ -1221,7 +1221,7 @@ static void dbg_restore_debug_regs(void) */ #ifdef CONFIG_X86_64 -void __cpuinit cpu_init(void) +void cpu_init(void) { struct orig_ist *oist; struct task_struct *me; @@ -1320,7 +1320,7 @@ void __cpuinit cpu_init(void) #else -void __cpuinit cpu_init(void) +void cpu_init(void) { int cpu = smp_processor_id(); struct task_struct *curr = current; diff --git a/arch/x86/kernel/cpu/cyrix.c b/arch/x86/kernel/cpu/cyrix.c index d048d5ca43c1c..f1fa2f2c2c906 100644 --- a/arch/x86/kernel/cpu/cyrix.c +++ b/arch/x86/kernel/cpu/cyrix.c @@ -15,7 +15,7 @@ /* * Read NSC/Cyrix DEVID registers (DIR) to get more detailed info. about the CPU */ -static void __cpuinit __do_cyrix_devid(unsigned char *dir0, unsigned char *dir1) +static void __do_cyrix_devid(unsigned char *dir0, unsigned char *dir1) { unsigned char ccr2, ccr3; @@ -44,7 +44,7 @@ static void __cpuinit __do_cyrix_devid(unsigned char *dir0, unsigned char *dir1) } } -static void __cpuinit do_cyrix_devid(unsigned char *dir0, unsigned char *dir1) +static void do_cyrix_devid(unsigned char *dir0, unsigned char *dir1) { unsigned long flags; @@ -59,25 +59,25 @@ static void __cpuinit do_cyrix_devid(unsigned char *dir0, unsigned char *dir1) * Actually since bugs.h doesn't even reference this perhaps someone should * fix the documentation ??? */ -static unsigned char Cx86_dir0_msb __cpuinitdata = 0; +static unsigned char Cx86_dir0_msb = 0; -static const char __cpuinitconst Cx86_model[][9] = { +static const char Cx86_model[][9] = { "Cx486", "Cx486", "5x86 ", "6x86", "MediaGX ", "6x86MX ", "M II ", "Unknown" }; -static const char __cpuinitconst Cx486_name[][5] = { +static const char Cx486_name[][5] = { "SLC", "DLC", "SLC2", "DLC2", "SRx", "DRx", "SRx2", "DRx2" }; -static const char __cpuinitconst Cx486S_name[][4] = { +static const char Cx486S_name[][4] = { "S", "S2", "Se", "S2e" }; -static const char __cpuinitconst Cx486D_name[][4] = { +static const char Cx486D_name[][4] = { "DX", "DX2", "?", "?", "?", "DX4" }; -static char Cx86_cb[] __cpuinitdata = "?.5x Core/Bus Clock"; -static const char __cpuinitconst cyrix_model_mult1[] = "12??43"; -static const char __cpuinitconst cyrix_model_mult2[] = "12233445"; +static char Cx86_cb[] = "?.5x Core/Bus Clock"; +static const char cyrix_model_mult1[] = "12??43"; +static const char cyrix_model_mult2[] = "12233445"; /* * Reset the slow-loop (SLOP) bit on the 686(L) which is set by some old @@ -87,7 +87,7 @@ static const char __cpuinitconst cyrix_model_mult2[] = "12233445"; * FIXME: our newer udelay uses the tsc. We don't need to frob with SLOP */ -static void __cpuinit check_cx686_slop(struct cpuinfo_x86 *c) +static void check_cx686_slop(struct cpuinfo_x86 *c) { unsigned long flags; @@ -112,7 +112,7 @@ static void __cpuinit check_cx686_slop(struct cpuinfo_x86 *c) } -static void __cpuinit set_cx86_reorder(void) +static void set_cx86_reorder(void) { u8 ccr3; @@ -127,7 +127,7 @@ static void __cpuinit set_cx86_reorder(void) setCx86(CX86_CCR3, ccr3); } -static void __cpuinit set_cx86_memwb(void) +static void set_cx86_memwb(void) { printk(KERN_INFO "Enable Memory-Write-back mode on Cyrix/NSC processor.\n"); @@ -143,7 +143,7 @@ static void __cpuinit set_cx86_memwb(void) * Configure later MediaGX and/or Geode processor. */ -static void __cpuinit geode_configure(void) +static void geode_configure(void) { unsigned long flags; u8 ccr3; @@ -166,7 +166,7 @@ static void __cpuinit geode_configure(void) local_irq_restore(flags); } -static void __cpuinit early_init_cyrix(struct cpuinfo_x86 *c) +static void early_init_cyrix(struct cpuinfo_x86 *c) { unsigned char dir0, dir0_msn, dir1 = 0; @@ -185,7 +185,7 @@ static void __cpuinit early_init_cyrix(struct cpuinfo_x86 *c) } } -static void __cpuinit init_cyrix(struct cpuinfo_x86 *c) +static void init_cyrix(struct cpuinfo_x86 *c) { unsigned char dir0, dir0_msn, dir0_lsn, dir1 = 0; char *buf = c->x86_model_id; @@ -356,7 +356,7 @@ static void __cpuinit init_cyrix(struct cpuinfo_x86 *c) /* * Handle National Semiconductor branded processors */ -static void __cpuinit init_nsc(struct cpuinfo_x86 *c) +static void init_nsc(struct cpuinfo_x86 *c) { /* * There may be GX1 processors in the wild that are branded @@ -405,7 +405,7 @@ static inline int test_cyrix_52div(void) return (unsigned char) (test >> 8) == 0x02; } -static void __cpuinit cyrix_identify(struct cpuinfo_x86 *c) +static void cyrix_identify(struct cpuinfo_x86 *c) { /* Detect Cyrix with disabled CPUID */ if (c->x86 == 4 && test_cyrix_52div()) { @@ -441,7 +441,7 @@ static void __cpuinit cyrix_identify(struct cpuinfo_x86 *c) } } -static const struct cpu_dev __cpuinitconst cyrix_cpu_dev = { +static const struct cpu_dev cyrix_cpu_dev = { .c_vendor = "Cyrix", .c_ident = { "CyrixInstead" }, .c_early_init = early_init_cyrix, @@ -452,7 +452,7 @@ static const struct cpu_dev __cpuinitconst cyrix_cpu_dev = { cpu_dev_register(cyrix_cpu_dev); -static const struct cpu_dev __cpuinitconst nsc_cpu_dev = { +static const struct cpu_dev nsc_cpu_dev = { .c_vendor = "NSC", .c_ident = { "Geode by NSC" }, .c_init = init_nsc, diff --git a/arch/x86/kernel/cpu/hypervisor.c b/arch/x86/kernel/cpu/hypervisor.c index 1e7e84a02ebaf..87279212d3186 100644 --- a/arch/x86/kernel/cpu/hypervisor.c +++ b/arch/x86/kernel/cpu/hypervisor.c @@ -60,7 +60,7 @@ detect_hypervisor_vendor(void) } } -void __cpuinit init_hypervisor(struct cpuinfo_x86 *c) +void init_hypervisor(struct cpuinfo_x86 *c) { if (x86_hyper && x86_hyper->set_cpu_features) x86_hyper->set_cpu_features(c); diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index 8533e69d2b89f..1d8152b764a7c 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -26,7 +26,7 @@ #include #endif -static void __cpuinit early_init_intel(struct cpuinfo_x86 *c) +static void early_init_intel(struct cpuinfo_x86 *c) { u64 misc_enable; @@ -178,7 +178,7 @@ static void __cpuinit early_init_intel(struct cpuinfo_x86 *c) * This is called before we do cpu ident work */ -int __cpuinit ppro_with_ram_bug(void) +int ppro_with_ram_bug(void) { /* Uses data from early_cpu_detect now */ if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && @@ -191,7 +191,7 @@ int __cpuinit ppro_with_ram_bug(void) return 0; } -static void __cpuinit intel_smp_check(struct cpuinfo_x86 *c) +static void intel_smp_check(struct cpuinfo_x86 *c) { /* calling is from identify_secondary_cpu() ? */ if (!c->cpu_index) @@ -211,7 +211,7 @@ static void __cpuinit intel_smp_check(struct cpuinfo_x86 *c) } } -static void __cpuinit intel_workarounds(struct cpuinfo_x86 *c) +static void intel_workarounds(struct cpuinfo_x86 *c) { unsigned long lo, hi; @@ -290,12 +290,12 @@ static void __cpuinit intel_workarounds(struct cpuinfo_x86 *c) intel_smp_check(c); } #else -static void __cpuinit intel_workarounds(struct cpuinfo_x86 *c) +static void intel_workarounds(struct cpuinfo_x86 *c) { } #endif -static void __cpuinit srat_detect_node(struct cpuinfo_x86 *c) +static void srat_detect_node(struct cpuinfo_x86 *c) { #ifdef CONFIG_NUMA unsigned node; @@ -315,7 +315,7 @@ static void __cpuinit srat_detect_node(struct cpuinfo_x86 *c) /* * find out the number of processor cores on the die */ -static int __cpuinit intel_num_cpu_cores(struct cpuinfo_x86 *c) +static int intel_num_cpu_cores(struct cpuinfo_x86 *c) { unsigned int eax, ebx, ecx, edx; @@ -330,7 +330,7 @@ static int __cpuinit intel_num_cpu_cores(struct cpuinfo_x86 *c) return 1; } -static void __cpuinit detect_vmx_virtcap(struct cpuinfo_x86 *c) +static void detect_vmx_virtcap(struct cpuinfo_x86 *c) { /* Intel VMX MSR indicated features */ #define X86_VMX_FEATURE_PROC_CTLS_TPR_SHADOW 0x00200000 @@ -368,7 +368,7 @@ static void __cpuinit detect_vmx_virtcap(struct cpuinfo_x86 *c) } } -static void __cpuinit init_intel(struct cpuinfo_x86 *c) +static void init_intel(struct cpuinfo_x86 *c) { unsigned int l2 = 0; @@ -488,7 +488,7 @@ static void __cpuinit init_intel(struct cpuinfo_x86 *c) } #ifdef CONFIG_X86_32 -static unsigned int __cpuinit intel_size_cache(struct cpuinfo_x86 *c, unsigned int size) +static unsigned int intel_size_cache(struct cpuinfo_x86 *c, unsigned int size) { /* * Intel PIII Tualatin. This comes in two flavours. @@ -522,7 +522,7 @@ static unsigned int __cpuinit intel_size_cache(struct cpuinfo_x86 *c, unsigned i #define STLB_4K 0x41 -static const struct _tlb_table intel_tlb_table[] __cpuinitconst = { +static const struct _tlb_table intel_tlb_table[] = { { 0x01, TLB_INST_4K, 32, " TLB_INST 4 KByte pages, 4-way set associative" }, { 0x02, TLB_INST_4M, 2, " TLB_INST 4 MByte pages, full associative" }, { 0x03, TLB_DATA_4K, 64, " TLB_DATA 4 KByte pages, 4-way set associative" }, @@ -552,7 +552,7 @@ static const struct _tlb_table intel_tlb_table[] __cpuinitconst = { { 0x00, 0, 0 } }; -static void __cpuinit intel_tlb_lookup(const unsigned char desc) +static void intel_tlb_lookup(const unsigned char desc) { unsigned char k; if (desc == 0) @@ -621,7 +621,7 @@ static void __cpuinit intel_tlb_lookup(const unsigned char desc) } } -static void __cpuinit intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c) +static void intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c) { switch ((c->x86 << 8) + c->x86_model) { case 0x60f: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */ @@ -650,7 +650,7 @@ static void __cpuinit intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c) } } -static void __cpuinit intel_detect_tlb(struct cpuinfo_x86 *c) +static void intel_detect_tlb(struct cpuinfo_x86 *c) { int i, j, n; unsigned int regs[4]; @@ -677,7 +677,7 @@ static void __cpuinit intel_detect_tlb(struct cpuinfo_x86 *c) intel_tlb_flushall_shift_set(c); } -static const struct cpu_dev __cpuinitconst intel_cpu_dev = { +static const struct cpu_dev intel_cpu_dev = { .c_vendor = "Intel", .c_ident = { "GenuineIntel" }, #ifdef CONFIG_X86_32 diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c index 7c6f7d548c0f9..3ef4c94a2f5eb 100644 --- a/arch/x86/kernel/cpu/intel_cacheinfo.c +++ b/arch/x86/kernel/cpu/intel_cacheinfo.c @@ -37,7 +37,7 @@ struct _cache_table { /* All the cache descriptor types we care about (no TLB or trace cache entries) */ -static const struct _cache_table __cpuinitconst cache_table[] = +static const struct _cache_table cache_table[] = { { 0x06, LVL_1_INST, 8 }, /* 4-way set assoc, 32 byte line size */ { 0x08, LVL_1_INST, 16 }, /* 4-way set assoc, 32 byte line size */ @@ -203,7 +203,7 @@ union l3_cache { unsigned val; }; -static const unsigned short __cpuinitconst assocs[] = { +static const unsigned short assocs[] = { [1] = 1, [2] = 2, [4] = 4, @@ -217,10 +217,10 @@ static const unsigned short __cpuinitconst assocs[] = { [0xf] = 0xffff /* fully associative - no way to show this currently */ }; -static const unsigned char __cpuinitconst levels[] = { 1, 1, 2, 3 }; -static const unsigned char __cpuinitconst types[] = { 1, 2, 3, 3 }; +static const unsigned char levels[] = { 1, 1, 2, 3 }; +static const unsigned char types[] = { 1, 2, 3, 3 }; -static void __cpuinit +static void amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax, union _cpuid4_leaf_ebx *ebx, union _cpuid4_leaf_ecx *ecx) @@ -302,7 +302,7 @@ struct _cache_attr { /* * L3 cache descriptors */ -static void __cpuinit amd_calc_l3_indices(struct amd_northbridge *nb) +static void amd_calc_l3_indices(struct amd_northbridge *nb) { struct amd_l3_cache *l3 = &nb->l3_cache; unsigned int sc0, sc1, sc2, sc3; @@ -325,7 +325,7 @@ static void __cpuinit amd_calc_l3_indices(struct amd_northbridge *nb) l3->indices = (max(max3(sc0, sc1, sc2), sc3) << 10) - 1; } -static void __cpuinit amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf, int index) +static void amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf, int index) { int node; @@ -528,7 +528,7 @@ static struct _cache_attr subcaches = #endif /* CONFIG_AMD_NB && CONFIG_SYSFS */ static int -__cpuinit cpuid4_cache_lookup_regs(int index, +cpuid4_cache_lookup_regs(int index, struct _cpuid4_info_regs *this_leaf) { union _cpuid4_leaf_eax eax; @@ -560,7 +560,7 @@ __cpuinit cpuid4_cache_lookup_regs(int index, return 0; } -static int __cpuinit find_num_cache_leaves(struct cpuinfo_x86 *c) +static int find_num_cache_leaves(struct cpuinfo_x86 *c) { unsigned int eax, ebx, ecx, edx, op; union _cpuid4_leaf_eax cache_eax; @@ -580,7 +580,7 @@ static int __cpuinit find_num_cache_leaves(struct cpuinfo_x86 *c) return i; } -void __cpuinit init_amd_cacheinfo(struct cpuinfo_x86 *c) +void init_amd_cacheinfo(struct cpuinfo_x86 *c) { if (cpu_has_topoext) { @@ -593,7 +593,7 @@ void __cpuinit init_amd_cacheinfo(struct cpuinfo_x86 *c) } } -unsigned int __cpuinit init_intel_cacheinfo(struct cpuinfo_x86 *c) +unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c) { /* Cache sizes */ unsigned int trace = 0, l1i = 0, l1d = 0, l2 = 0, l3 = 0; @@ -746,7 +746,7 @@ static DEFINE_PER_CPU(struct _cpuid4_info *, ici_cpuid4_info); #ifdef CONFIG_SMP -static int __cpuinit cache_shared_amd_cpu_map_setup(unsigned int cpu, int index) +static int cache_shared_amd_cpu_map_setup(unsigned int cpu, int index) { struct _cpuid4_info *this_leaf; int i, sibling; @@ -795,7 +795,7 @@ static int __cpuinit cache_shared_amd_cpu_map_setup(unsigned int cpu, int index) return 1; } -static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index) +static void cache_shared_cpu_map_setup(unsigned int cpu, int index) { struct _cpuid4_info *this_leaf, *sibling_leaf; unsigned long num_threads_sharing; @@ -830,7 +830,7 @@ static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index) } } } -static void __cpuinit cache_remove_shared_cpu_map(unsigned int cpu, int index) +static void cache_remove_shared_cpu_map(unsigned int cpu, int index) { struct _cpuid4_info *this_leaf, *sibling_leaf; int sibling; @@ -843,16 +843,16 @@ static void __cpuinit cache_remove_shared_cpu_map(unsigned int cpu, int index) } } #else -static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index) +static void cache_shared_cpu_map_setup(unsigned int cpu, int index) { } -static void __cpuinit cache_remove_shared_cpu_map(unsigned int cpu, int index) +static void cache_remove_shared_cpu_map(unsigned int cpu, int index) { } #endif -static void __cpuinit free_cache_attributes(unsigned int cpu) +static void free_cache_attributes(unsigned int cpu) { int i; @@ -863,7 +863,7 @@ static void __cpuinit free_cache_attributes(unsigned int cpu) per_cpu(ici_cpuid4_info, cpu) = NULL; } -static void __cpuinit get_cpu_leaves(void *_retval) +static void get_cpu_leaves(void *_retval) { int j, *retval = _retval, cpu = smp_processor_id(); @@ -883,7 +883,7 @@ static void __cpuinit get_cpu_leaves(void *_retval) } } -static int __cpuinit detect_cache_attributes(unsigned int cpu) +static int detect_cache_attributes(unsigned int cpu) { int retval; @@ -1017,7 +1017,7 @@ static struct attribute *default_attrs[] = { }; #ifdef CONFIG_AMD_NB -static struct attribute ** __cpuinit amd_l3_attrs(void) +static struct attribute ** amd_l3_attrs(void) { static struct attribute **attrs; int n; @@ -1093,7 +1093,7 @@ static struct kobj_type ktype_percpu_entry = { .sysfs_ops = &sysfs_ops, }; -static void __cpuinit cpuid4_cache_sysfs_exit(unsigned int cpu) +static void cpuid4_cache_sysfs_exit(unsigned int cpu) { kfree(per_cpu(ici_cache_kobject, cpu)); kfree(per_cpu(ici_index_kobject, cpu)); @@ -1102,7 +1102,7 @@ static void __cpuinit cpuid4_cache_sysfs_exit(unsigned int cpu) free_cache_attributes(cpu); } -static int __cpuinit cpuid4_cache_sysfs_init(unsigned int cpu) +static int cpuid4_cache_sysfs_init(unsigned int cpu) { int err; @@ -1134,7 +1134,7 @@ static int __cpuinit cpuid4_cache_sysfs_init(unsigned int cpu) static DECLARE_BITMAP(cache_dev_map, NR_CPUS); /* Add/Remove cache interface for CPU device */ -static int __cpuinit cache_add_dev(struct device *dev) +static int cache_add_dev(struct device *dev) { unsigned int cpu = dev->id; unsigned long i, j; @@ -1185,7 +1185,7 @@ static int __cpuinit cache_add_dev(struct device *dev) return 0; } -static void __cpuinit cache_remove_dev(struct device *dev) +static void cache_remove_dev(struct device *dev) { unsigned int cpu = dev->id; unsigned long i; @@ -1202,7 +1202,7 @@ static void __cpuinit cache_remove_dev(struct device *dev) cpuid4_cache_sysfs_exit(cpu); } -static int __cpuinit cacheinfo_cpu_callback(struct notifier_block *nfb, +static int cacheinfo_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; @@ -1222,7 +1222,7 @@ static int __cpuinit cacheinfo_cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata cacheinfo_cpu_notifier = { +static struct notifier_block cacheinfo_cpu_notifier = { .notifier_call = cacheinfo_cpu_callback, }; diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index 9239504b41cbc..a33bca5083fb7 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -1360,7 +1360,7 @@ int mce_notify_irq(void) } EXPORT_SYMBOL_GPL(mce_notify_irq); -static int __cpuinit __mcheck_cpu_mce_banks_init(void) +static int __mcheck_cpu_mce_banks_init(void) { int i; u8 num_banks = mca_cfg.banks; @@ -1381,7 +1381,7 @@ static int __cpuinit __mcheck_cpu_mce_banks_init(void) /* * Initialize Machine Checks for a CPU. */ -static int __cpuinit __mcheck_cpu_cap_init(void) +static int __mcheck_cpu_cap_init(void) { unsigned b; u64 cap; @@ -1480,7 +1480,7 @@ static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs) } /* Add per CPU specific workarounds here */ -static int __cpuinit __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c) +static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c) { struct mca_config *cfg = &mca_cfg; @@ -1590,7 +1590,7 @@ static int __cpuinit __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c) return 0; } -static int __cpuinit __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c) +static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c) { if (c->x86 != 5) return 0; @@ -1661,7 +1661,7 @@ void (*machine_check_vector)(struct pt_regs *, long error_code) = * Called for each booted CPU to set up machine checks. * Must be called with preempt off: */ -void __cpuinit mcheck_cpu_init(struct cpuinfo_x86 *c) +void mcheck_cpu_init(struct cpuinfo_x86 *c) { if (mca_cfg.disabled) return; @@ -2225,7 +2225,7 @@ static void mce_device_release(struct device *dev) } /* Per cpu device init. All of the cpus still share the same ctrl bank: */ -static __cpuinit int mce_device_create(unsigned int cpu) +static int mce_device_create(unsigned int cpu) { struct device *dev; int err; @@ -2271,7 +2271,7 @@ static __cpuinit int mce_device_create(unsigned int cpu) return err; } -static __cpuinit void mce_device_remove(unsigned int cpu) +static void mce_device_remove(unsigned int cpu) { struct device *dev = per_cpu(mce_device, cpu); int i; @@ -2291,7 +2291,7 @@ static __cpuinit void mce_device_remove(unsigned int cpu) } /* Make sure there are no machine checks on offlined CPUs. */ -static void __cpuinit mce_disable_cpu(void *h) +static void mce_disable_cpu(void *h) { unsigned long action = *(unsigned long *)h; int i; @@ -2309,7 +2309,7 @@ static void __cpuinit mce_disable_cpu(void *h) } } -static void __cpuinit mce_reenable_cpu(void *h) +static void mce_reenable_cpu(void *h) { unsigned long action = *(unsigned long *)h; int i; @@ -2328,7 +2328,7 @@ static void __cpuinit mce_reenable_cpu(void *h) } /* Get notified when a cpu comes on/off. Be hotplug friendly. */ -static int __cpuinit +static int mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; @@ -2364,7 +2364,7 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) return NOTIFY_OK; } -static struct notifier_block mce_cpu_notifier __cpuinitdata = { +static struct notifier_block mce_cpu_notifier = { .notifier_call = mce_cpu_callback, }; diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c index 338a4ae486bc8..1246fa656739a 100644 --- a/arch/x86/kernel/cpu/mcheck/mce_amd.c +++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c @@ -458,7 +458,7 @@ static struct kobj_type threshold_ktype = { .default_attrs = default_attrs, }; -static __cpuinit int allocate_threshold_blocks(unsigned int cpu, +static int allocate_threshold_blocks(unsigned int cpu, unsigned int bank, unsigned int block, u32 address) @@ -543,7 +543,7 @@ static __cpuinit int allocate_threshold_blocks(unsigned int cpu, return err; } -static __cpuinit int __threshold_add_blocks(struct threshold_bank *b) +static int __threshold_add_blocks(struct threshold_bank *b) { struct list_head *head = &b->blocks->miscj; struct threshold_block *pos = NULL; @@ -567,7 +567,7 @@ static __cpuinit int __threshold_add_blocks(struct threshold_bank *b) return err; } -static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank) +static int threshold_create_bank(unsigned int cpu, unsigned int bank) { struct device *dev = per_cpu(mce_device, cpu); struct amd_northbridge *nb = NULL; @@ -632,7 +632,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank) } /* create dir/files for all valid threshold banks */ -static __cpuinit int threshold_create_device(unsigned int cpu) +static int threshold_create_device(unsigned int cpu) { unsigned int bank; struct threshold_bank **bp; @@ -736,7 +736,7 @@ static void threshold_remove_device(unsigned int cpu) } /* get notified when a cpu comes on/off */ -static void __cpuinit +static void amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu) { switch (action) { diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c index 47a1870279aad..7b7e8fdd771cb 100644 --- a/arch/x86/kernel/cpu/mcheck/therm_throt.c +++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c @@ -221,7 +221,7 @@ static int thresh_event_valid(int event) #ifdef CONFIG_SYSFS /* Add/Remove thermal_throttle interface for CPU device: */ -static __cpuinit int thermal_throttle_add_dev(struct device *dev, +static int thermal_throttle_add_dev(struct device *dev, unsigned int cpu) { int err; @@ -248,7 +248,7 @@ static __cpuinit int thermal_throttle_add_dev(struct device *dev, return err; } -static __cpuinit void thermal_throttle_remove_dev(struct device *dev) +static void thermal_throttle_remove_dev(struct device *dev) { sysfs_remove_group(&dev->kobj, &thermal_attr_group); } @@ -257,7 +257,7 @@ static __cpuinit void thermal_throttle_remove_dev(struct device *dev) static DEFINE_MUTEX(therm_cpu_lock); /* Get notified when a cpu comes on/off. Be hotplug friendly. */ -static __cpuinit int +static int thermal_throttle_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) @@ -288,7 +288,7 @@ thermal_throttle_cpu_callback(struct notifier_block *nfb, return notifier_from_errno(err); } -static struct notifier_block thermal_throttle_cpu_notifier __cpuinitdata = +static struct notifier_block thermal_throttle_cpu_notifier = { .notifier_call = thermal_throttle_cpu_callback, }; diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index 123d9e2271dc5..715528dd6d3c8 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c @@ -1271,7 +1271,7 @@ perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) struct event_constraint emptyconstraint; struct event_constraint unconstrained; -static int __cpuinit +static int x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (long)hcpu; diff --git a/arch/x86/kernel/cpu/perf_event_amd_ibs.c b/arch/x86/kernel/cpu/perf_event_amd_ibs.c index b46601ada8134..4b8e4d3cd6ea6 100644 --- a/arch/x86/kernel/cpu/perf_event_amd_ibs.c +++ b/arch/x86/kernel/cpu/perf_event_amd_ibs.c @@ -894,7 +894,7 @@ static inline void perf_ibs_pm_init(void) { } #endif -static int __cpuinit +static int perf_ibs_cpu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { switch (action & ~CPU_TASKS_FROZEN) { diff --git a/arch/x86/kernel/cpu/perf_event_amd_uncore.c b/arch/x86/kernel/cpu/perf_event_amd_uncore.c index c0c661adf03e2..b7b2e3de95e39 100644 --- a/arch/x86/kernel/cpu/perf_event_amd_uncore.c +++ b/arch/x86/kernel/cpu/perf_event_amd_uncore.c @@ -288,13 +288,13 @@ static struct pmu amd_l2_pmu = { .read = amd_uncore_read, }; -static struct amd_uncore * __cpuinit amd_uncore_alloc(unsigned int cpu) +static struct amd_uncore * amd_uncore_alloc(unsigned int cpu) { return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL, cpu_to_node(cpu)); } -static void __cpuinit amd_uncore_cpu_up_prepare(unsigned int cpu) +static void amd_uncore_cpu_up_prepare(unsigned int cpu) { struct amd_uncore *uncore; @@ -322,7 +322,7 @@ static void __cpuinit amd_uncore_cpu_up_prepare(unsigned int cpu) } static struct amd_uncore * -__cpuinit amd_uncore_find_online_sibling(struct amd_uncore *this, +amd_uncore_find_online_sibling(struct amd_uncore *this, struct amd_uncore * __percpu *uncores) { unsigned int cpu; @@ -348,7 +348,7 @@ __cpuinit amd_uncore_find_online_sibling(struct amd_uncore *this, return this; } -static void __cpuinit amd_uncore_cpu_starting(unsigned int cpu) +static void amd_uncore_cpu_starting(unsigned int cpu) { unsigned int eax, ebx, ecx, edx; struct amd_uncore *uncore; @@ -376,7 +376,7 @@ static void __cpuinit amd_uncore_cpu_starting(unsigned int cpu) } } -static void __cpuinit uncore_online(unsigned int cpu, +static void uncore_online(unsigned int cpu, struct amd_uncore * __percpu *uncores) { struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu); @@ -388,7 +388,7 @@ static void __cpuinit uncore_online(unsigned int cpu, cpumask_set_cpu(cpu, uncore->active_mask); } -static void __cpuinit amd_uncore_cpu_online(unsigned int cpu) +static void amd_uncore_cpu_online(unsigned int cpu) { if (amd_uncore_nb) uncore_online(cpu, amd_uncore_nb); @@ -397,7 +397,7 @@ static void __cpuinit amd_uncore_cpu_online(unsigned int cpu) uncore_online(cpu, amd_uncore_l2); } -static void __cpuinit uncore_down_prepare(unsigned int cpu, +static void uncore_down_prepare(unsigned int cpu, struct amd_uncore * __percpu *uncores) { unsigned int i; @@ -423,7 +423,7 @@ static void __cpuinit uncore_down_prepare(unsigned int cpu, } } -static void __cpuinit amd_uncore_cpu_down_prepare(unsigned int cpu) +static void amd_uncore_cpu_down_prepare(unsigned int cpu) { if (amd_uncore_nb) uncore_down_prepare(cpu, amd_uncore_nb); @@ -432,7 +432,7 @@ static void __cpuinit amd_uncore_cpu_down_prepare(unsigned int cpu) uncore_down_prepare(cpu, amd_uncore_l2); } -static void __cpuinit uncore_dead(unsigned int cpu, +static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores) { struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu); @@ -445,7 +445,7 @@ static void __cpuinit uncore_dead(unsigned int cpu, *per_cpu_ptr(amd_uncore_nb, cpu) = NULL; } -static void __cpuinit amd_uncore_cpu_dead(unsigned int cpu) +static void amd_uncore_cpu_dead(unsigned int cpu) { if (amd_uncore_nb) uncore_dead(cpu, amd_uncore_nb); @@ -454,7 +454,7 @@ static void __cpuinit amd_uncore_cpu_dead(unsigned int cpu) uncore_dead(cpu, amd_uncore_l2); } -static int __cpuinit +static int amd_uncore_cpu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { @@ -489,7 +489,7 @@ amd_uncore_cpu_notifier(struct notifier_block *self, unsigned long action, return NOTIFY_OK; } -static struct notifier_block amd_uncore_cpu_notifier_block __cpuinitdata = { +static struct notifier_block amd_uncore_cpu_notifier_block = { .notifier_call = amd_uncore_cpu_notifier, .priority = CPU_PRI_PERF + 1, }; diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.c b/arch/x86/kernel/cpu/perf_event_intel_uncore.c index 7185af255fb52..afaf184259b94 100644 --- a/arch/x86/kernel/cpu/perf_event_intel_uncore.c +++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.c @@ -3311,7 +3311,7 @@ static void __init uncore_pci_exit(void) /* CPU hot plug/unplug are serialized by cpu_add_remove_lock mutex */ static LIST_HEAD(boxes_to_free); -static void __cpuinit uncore_kfree_boxes(void) +static void uncore_kfree_boxes(void) { struct intel_uncore_box *box; @@ -3323,7 +3323,7 @@ static void __cpuinit uncore_kfree_boxes(void) } } -static void __cpuinit uncore_cpu_dying(int cpu) +static void uncore_cpu_dying(int cpu) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; @@ -3342,7 +3342,7 @@ static void __cpuinit uncore_cpu_dying(int cpu) } } -static int __cpuinit uncore_cpu_starting(int cpu) +static int uncore_cpu_starting(int cpu) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; @@ -3385,7 +3385,7 @@ static int __cpuinit uncore_cpu_starting(int cpu) return 0; } -static int __cpuinit uncore_cpu_prepare(int cpu, int phys_id) +static int uncore_cpu_prepare(int cpu, int phys_id) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; @@ -3411,7 +3411,7 @@ static int __cpuinit uncore_cpu_prepare(int cpu, int phys_id) return 0; } -static void __cpuinit +static void uncore_change_context(struct intel_uncore_type **uncores, int old_cpu, int new_cpu) { struct intel_uncore_type *type; @@ -3449,7 +3449,7 @@ uncore_change_context(struct intel_uncore_type **uncores, int old_cpu, int new_c } } -static void __cpuinit uncore_event_exit_cpu(int cpu) +static void uncore_event_exit_cpu(int cpu) { int i, phys_id, target; @@ -3477,7 +3477,7 @@ static void __cpuinit uncore_event_exit_cpu(int cpu) uncore_change_context(pci_uncores, cpu, target); } -static void __cpuinit uncore_event_init_cpu(int cpu) +static void uncore_event_init_cpu(int cpu) { int i, phys_id; @@ -3494,7 +3494,7 @@ static void __cpuinit uncore_event_init_cpu(int cpu) } static int - __cpuinit uncore_cpu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) + uncore_cpu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (long)hcpu; @@ -3534,7 +3534,7 @@ static int return NOTIFY_OK; } -static struct notifier_block uncore_cpu_nb __cpuinitdata = { +static struct notifier_block uncore_cpu_nb = { .notifier_call = uncore_cpu_notifier, /* * to migrate uncore events, our notifier should be executed diff --git a/arch/x86/kernel/cpu/rdrand.c b/arch/x86/kernel/cpu/rdrand.c index feca286c2bb4a..88db010845cb2 100644 --- a/arch/x86/kernel/cpu/rdrand.c +++ b/arch/x86/kernel/cpu/rdrand.c @@ -52,7 +52,7 @@ static inline int rdrand_long(unsigned long *v) */ #define RESEED_LOOP ((512*128)/sizeof(unsigned long)) -void __cpuinit x86_init_rdrand(struct cpuinfo_x86 *c) +void x86_init_rdrand(struct cpuinfo_x86 *c) { #ifdef CONFIG_ARCH_RANDOM unsigned long tmp; diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index d92b5dad15dd4..f2cc63e9cf08f 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -24,13 +24,13 @@ enum cpuid_regs { CR_EBX }; -void __cpuinit init_scattered_cpuid_features(struct cpuinfo_x86 *c) +void init_scattered_cpuid_features(struct cpuinfo_x86 *c) { u32 max_level; u32 regs[4]; const struct cpuid_bit *cb; - static const struct cpuid_bit __cpuinitconst cpuid_bits[] = { + static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_DTHERM, CR_EAX, 0, 0x00000006, 0 }, { X86_FEATURE_IDA, CR_EAX, 1, 0x00000006, 0 }, { X86_FEATURE_ARAT, CR_EAX, 2, 0x00000006, 0 }, diff --git a/arch/x86/kernel/cpu/topology.c b/arch/x86/kernel/cpu/topology.c index 4397e987a1cfb..4c60eaf0571c2 100644 --- a/arch/x86/kernel/cpu/topology.c +++ b/arch/x86/kernel/cpu/topology.c @@ -26,7 +26,7 @@ * exists, use it for populating initial_apicid and cpu topology * detection. */ -void __cpuinit detect_extended_topology(struct cpuinfo_x86 *c) +void detect_extended_topology(struct cpuinfo_x86 *c) { #ifdef CONFIG_SMP unsigned int eax, ebx, ecx, edx, sub_index; diff --git a/arch/x86/kernel/cpu/transmeta.c b/arch/x86/kernel/cpu/transmeta.c index 28000743bbb06..aa0430d69b900 100644 --- a/arch/x86/kernel/cpu/transmeta.c +++ b/arch/x86/kernel/cpu/transmeta.c @@ -5,7 +5,7 @@ #include #include "cpu.h" -static void __cpuinit early_init_transmeta(struct cpuinfo_x86 *c) +static void early_init_transmeta(struct cpuinfo_x86 *c) { u32 xlvl; @@ -17,7 +17,7 @@ static void __cpuinit early_init_transmeta(struct cpuinfo_x86 *c) } } -static void __cpuinit init_transmeta(struct cpuinfo_x86 *c) +static void init_transmeta(struct cpuinfo_x86 *c) { unsigned int cap_mask, uk, max, dummy; unsigned int cms_rev1, cms_rev2; @@ -98,7 +98,7 @@ static void __cpuinit init_transmeta(struct cpuinfo_x86 *c) #endif } -static const struct cpu_dev __cpuinitconst transmeta_cpu_dev = { +static const struct cpu_dev transmeta_cpu_dev = { .c_vendor = "Transmeta", .c_ident = { "GenuineTMx86", "TransmetaCPU" }, .c_early_init = early_init_transmeta, diff --git a/arch/x86/kernel/cpu/umc.c b/arch/x86/kernel/cpu/umc.c index fd2c37bf7acb6..202759a141212 100644 --- a/arch/x86/kernel/cpu/umc.c +++ b/arch/x86/kernel/cpu/umc.c @@ -8,7 +8,7 @@ * so no special init takes place. */ -static const struct cpu_dev __cpuinitconst umc_cpu_dev = { +static const struct cpu_dev umc_cpu_dev = { .c_vendor = "UMC", .c_ident = { "UMC UMC UMC" }, .c_models = { diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c index 03a36321ec544..7076878404ec0 100644 --- a/arch/x86/kernel/cpu/vmware.c +++ b/arch/x86/kernel/cpu/vmware.c @@ -122,7 +122,7 @@ static bool __init vmware_platform(void) * so that the kernel could just trust the hypervisor with providing a * reliable virtual TSC that is suitable for timekeeping. */ -static void __cpuinit vmware_set_cpu_features(struct cpuinfo_x86 *c) +static void vmware_set_cpu_features(struct cpuinfo_x86 *c) { set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC); set_cpu_cap(c, X86_FEATURE_TSC_RELIABLE); diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c index 1e4dbcfe6d316..563fea5fcba26 100644 --- a/arch/x86/kernel/cpuid.c +++ b/arch/x86/kernel/cpuid.c @@ -137,7 +137,7 @@ static const struct file_operations cpuid_fops = { .open = cpuid_open, }; -static __cpuinit int cpuid_device_create(int cpu) +static int cpuid_device_create(int cpu) { struct device *dev; @@ -151,7 +151,7 @@ static void cpuid_device_destroy(int cpu) device_destroy(cpuid_class, MKDEV(CPUID_MAJOR, cpu)); } -static int __cpuinit cpuid_class_cpu_callback(struct notifier_block *nfb, +static int cpuid_class_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { diff --git a/arch/x86/kernel/devicetree.c b/arch/x86/kernel/devicetree.c index 2fbad6b9f23c0..a4aad73cd57f3 100644 --- a/arch/x86/kernel/devicetree.c +++ b/arch/x86/kernel/devicetree.c @@ -132,7 +132,7 @@ static void x86_of_pci_irq_disable(struct pci_dev *dev) { } -void __cpuinit x86_of_pci_init(void) +void x86_of_pci_init(void) { pcibios_enable_irq = x86_of_pci_irq_enable; pcibios_disable_irq = x86_of_pci_irq_disable; diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c index b03ff1842547a..e6f08c6853f7c 100644 --- a/arch/x86/kernel/i387.c +++ b/arch/x86/kernel/i387.c @@ -119,7 +119,7 @@ unsigned int xstate_size; EXPORT_SYMBOL_GPL(xstate_size); static struct i387_fxsave_struct fx_scratch __cpuinitdata; -static void __cpuinit mxcsr_feature_mask_init(void) +static void mxcsr_feature_mask_init(void) { unsigned long mask = 0; @@ -133,7 +133,7 @@ static void __cpuinit mxcsr_feature_mask_init(void) mxcsr_feature_mask &= mask; } -static void __cpuinit init_thread_xstate(void) +static void init_thread_xstate(void) { /* * Note that xstate_size might be overwriten later during @@ -162,7 +162,7 @@ static void __cpuinit init_thread_xstate(void) * into all processes. */ -void __cpuinit fpu_init(void) +void fpu_init(void) { unsigned long cr0; unsigned long cr4_mask = 0; diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c index 344faf8d0d621..4186755f1d7cf 100644 --- a/arch/x86/kernel/irq_32.c +++ b/arch/x86/kernel/irq_32.c @@ -119,7 +119,7 @@ execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq) /* * allocate per-cpu stacks for hardirq and for softirq processing */ -void __cpuinit irq_ctx_init(int cpu) +void irq_ctx_init(int cpu) { union irq_ctx *irqctx; diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index c4ff2a9161399..d561f0a5b33a9 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -327,7 +327,7 @@ static void kvm_guest_apic_eoi_write(u32 reg, u32 val) apic_write(APIC_EOI, APIC_EOI_ACK); } -void __cpuinit kvm_guest_cpu_init(void) +void kvm_guest_cpu_init(void) { if (!kvm_para_available()) return; @@ -428,7 +428,7 @@ static void __init kvm_smp_prepare_boot_cpu(void) native_smp_prepare_boot_cpu(); } -static void __cpuinit kvm_guest_cpu_online(void *dummy) +static void kvm_guest_cpu_online(void *dummy) { kvm_guest_cpu_init(); } @@ -442,7 +442,7 @@ static void kvm_guest_cpu_offline(void *dummy) apf_task_wake_all(); } -static int __cpuinit kvm_cpu_notify(struct notifier_block *self, +static int kvm_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { int cpu = (unsigned long)hcpu; @@ -462,7 +462,7 @@ static int __cpuinit kvm_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata kvm_cpu_notifier = { +static struct notifier_block kvm_cpu_notifier = { .notifier_call = kvm_cpu_notify, }; #endif diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index 41514f56c2413..836c78b7ac1d0 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -185,7 +185,7 @@ static void kvm_restore_sched_clock_state(void) } #ifdef CONFIG_X86_LOCAL_APIC -static void __cpuinit kvm_setup_secondary_clock(void) +static void kvm_setup_secondary_clock(void) { /* * Now that the first cpu already had this clocksource initialized, diff --git a/arch/x86/kernel/microcode_core.c b/arch/x86/kernel/microcode_core.c index 22db92bbdf1a4..15c987698b0f7 100644 --- a/arch/x86/kernel/microcode_core.c +++ b/arch/x86/kernel/microcode_core.c @@ -468,7 +468,7 @@ static struct syscore_ops mc_syscore_ops = { .resume = mc_bp_resume, }; -static __cpuinit int +static int mc_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; diff --git a/arch/x86/kernel/microcode_core_early.c b/arch/x86/kernel/microcode_core_early.c index 833d51d6ee065..c97844e3aedbb 100644 --- a/arch/x86/kernel/microcode_core_early.c +++ b/arch/x86/kernel/microcode_core_early.c @@ -40,7 +40,7 @@ * * x86_vendor() gets vendor information directly through cpuid. */ -static int __cpuinit x86_vendor(void) +static int x86_vendor(void) { u32 eax = 0x00000000; u32 ebx, ecx = 0, edx; @@ -56,7 +56,7 @@ static int __cpuinit x86_vendor(void) return X86_VENDOR_UNKNOWN; } -static int __cpuinit x86_family(void) +static int x86_family(void) { u32 eax = 0x00000001; u32 ebx, ecx = 0, edx; @@ -85,7 +85,7 @@ void __init load_ucode_bsp(void) load_ucode_intel_bsp(); } -void __cpuinit load_ucode_ap(void) +void load_ucode_ap(void) { int vendor, x86; diff --git a/arch/x86/kernel/microcode_intel_early.c b/arch/x86/kernel/microcode_intel_early.c index a883942aee44a..a1f9ebf459165 100644 --- a/arch/x86/kernel/microcode_intel_early.c +++ b/arch/x86/kernel/microcode_intel_early.c @@ -34,7 +34,7 @@ struct mc_saved_data { struct microcode_intel **mc_saved; } mc_saved_data; -static enum ucode_state __cpuinit +static enum ucode_state generic_load_microcode_early(struct microcode_intel **mc_saved_p, unsigned int mc_saved_count, struct ucode_cpu_info *uci) @@ -69,7 +69,7 @@ generic_load_microcode_early(struct microcode_intel **mc_saved_p, return state; } -static void __cpuinit +static void microcode_pointer(struct microcode_intel **mc_saved, unsigned long *mc_saved_in_initrd, unsigned long initrd_start, int mc_saved_count) @@ -82,7 +82,7 @@ microcode_pointer(struct microcode_intel **mc_saved, } #ifdef CONFIG_X86_32 -static void __cpuinit +static void microcode_phys(struct microcode_intel **mc_saved_tmp, struct mc_saved_data *mc_saved_data) { @@ -101,7 +101,7 @@ microcode_phys(struct microcode_intel **mc_saved_tmp, } #endif -static enum ucode_state __cpuinit +static enum ucode_state load_microcode(struct mc_saved_data *mc_saved_data, unsigned long *mc_saved_in_initrd, unsigned long initrd_start, @@ -375,7 +375,7 @@ do { \ #define native_wrmsr(msr, low, high) \ native_write_msr(msr, low, high); -static int __cpuinit collect_cpu_info_early(struct ucode_cpu_info *uci) +static int collect_cpu_info_early(struct ucode_cpu_info *uci) { unsigned int val[2]; u8 x86, x86_model; @@ -584,7 +584,7 @@ scan_microcode(unsigned long start, unsigned long end, /* * Print ucode update info. */ -static void __cpuinit +static void print_ucode_info(struct ucode_cpu_info *uci, unsigned int date) { int cpu = smp_processor_id(); @@ -605,7 +605,7 @@ static int current_mc_date; /* * Print early updated ucode info after printk works. This is delayed info dump. */ -void __cpuinit show_ucode_info_early(void) +void show_ucode_info_early(void) { struct ucode_cpu_info uci; @@ -621,7 +621,7 @@ void __cpuinit show_ucode_info_early(void) * mc_saved_data.mc_saved and delay printing microcode info in * show_ucode_info_early() until printk() works. */ -static void __cpuinit print_ucode(struct ucode_cpu_info *uci) +static void print_ucode(struct ucode_cpu_info *uci) { struct microcode_intel *mc_intel; int *delay_ucode_info_p; @@ -643,12 +643,12 @@ static void __cpuinit print_ucode(struct ucode_cpu_info *uci) * Flush global tlb. We only do this in x86_64 where paging has been enabled * already and PGE should be enabled as well. */ -static inline void __cpuinit flush_tlb_early(void) +static inline void flush_tlb_early(void) { __native_flush_tlb_global_irq_disabled(); } -static inline void __cpuinit print_ucode(struct ucode_cpu_info *uci) +static inline void print_ucode(struct ucode_cpu_info *uci) { struct microcode_intel *mc_intel; @@ -660,7 +660,7 @@ static inline void __cpuinit print_ucode(struct ucode_cpu_info *uci) } #endif -static int __cpuinit apply_microcode_early(struct mc_saved_data *mc_saved_data, +static int apply_microcode_early(struct mc_saved_data *mc_saved_data, struct ucode_cpu_info *uci) { struct microcode_intel *mc_intel; @@ -763,7 +763,7 @@ load_ucode_intel_bsp(void) #endif } -void __cpuinit load_ucode_intel_ap(void) +void load_ucode_intel_ap(void) { struct mc_saved_data *mc_saved_data_p; struct ucode_cpu_info uci; diff --git a/arch/x86/kernel/mmconf-fam10h_64.c b/arch/x86/kernel/mmconf-fam10h_64.c index ac861b8348e2d..33fd4e6407b7b 100644 --- a/arch/x86/kernel/mmconf-fam10h_64.c +++ b/arch/x86/kernel/mmconf-fam10h_64.c @@ -24,14 +24,14 @@ struct pci_hostbridge_probe { u32 device; }; -static u64 __cpuinitdata fam10h_pci_mmconf_base; +static u64 fam10h_pci_mmconf_base; -static struct pci_hostbridge_probe pci_probes[] __cpuinitdata = { +static struct pci_hostbridge_probe pci_probes[] = { { 0, 0x18, PCI_VENDOR_ID_AMD, 0x1200 }, { 0xff, 0, PCI_VENDOR_ID_AMD, 0x1200 }, }; -static int __cpuinit cmp_range(const void *x1, const void *x2) +static int cmp_range(const void *x1, const void *x2) { const struct range *r1 = x1; const struct range *r2 = x2; @@ -49,7 +49,7 @@ static int __cpuinit cmp_range(const void *x1, const void *x2) /* need to avoid (0xfd<<32), (0xfe<<32), and (0xff<<32), ht used space */ #define FAM10H_PCI_MMCONF_BASE (0xfcULL<<32) #define BASE_VALID(b) ((b) + MMCONF_SIZE <= (0xfdULL<<32) || (b) >= (1ULL<<40)) -static void __cpuinit get_fam10h_pci_mmconf_base(void) +static void get_fam10h_pci_mmconf_base(void) { int i; unsigned bus; @@ -166,7 +166,7 @@ static void __cpuinit get_fam10h_pci_mmconf_base(void) fam10h_pci_mmconf_base = base; } -void __cpuinit fam10h_check_enable_mmcfg(void) +void fam10h_check_enable_mmcfg(void) { u64 val; u32 address; @@ -230,7 +230,7 @@ static const struct dmi_system_id __initconst mmconf_dmi_table[] = { {} }; -/* Called from a __cpuinit function, but only on the BSP. */ +/* Called from a function, but only on the BSP. */ void __ref check_enable_amd_mmconf_dmi(void) { dmi_check_system(mmconf_dmi_table); diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c index ce130493b8020..6d9d08e443c57 100644 --- a/arch/x86/kernel/msr.c +++ b/arch/x86/kernel/msr.c @@ -200,7 +200,7 @@ static const struct file_operations msr_fops = { .compat_ioctl = msr_ioctl, }; -static int __cpuinit msr_device_create(int cpu) +static int msr_device_create(int cpu) { struct device *dev; @@ -214,7 +214,7 @@ static void msr_device_destroy(int cpu) device_destroy(msr_class, MKDEV(MSR_MAJOR, cpu)); } -static int __cpuinit msr_class_cpu_callback(struct notifier_block *nfb, +static int msr_class_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 48f4399534361..3179fc1bfd970 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -385,7 +385,7 @@ static void amd_e400_idle(void) default_idle(); } -void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c) +void select_idle_routine(const struct cpuinfo_x86 *c) { #ifdef CONFIG_SMP if (boot_option_idle_override == IDLE_POLL && smp_num_siblings > 1) diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index e4baa389ee502..8aac112426baa 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -170,7 +170,7 @@ static struct resource bss_resource = { #ifdef CONFIG_X86_32 /* cpu data as detected by the assembly code in head.S */ -struct cpuinfo_x86 new_cpu_data __cpuinitdata = { +struct cpuinfo_x86 new_cpu_data = { .wp_works_ok = -1, }; /* common cpu data for all cpus */ diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 87084ab90d190..009e1adaf9073 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -130,7 +130,7 @@ atomic_t init_deasserted; * Report back to the Boot Processor during boot time or to the caller processor * during CPU online. */ -static void __cpuinit smp_callin(void) +static void smp_callin(void) { int cpuid, phys_id; unsigned long timeout; @@ -237,7 +237,7 @@ static int enable_start_cpu0; /* * Activate a secondary processor. */ -notrace static void __cpuinit start_secondary(void *unused) +notrace static void start_secondary(void *unused) { /* * Don't put *anything* before cpu_init(), SMP booting is too @@ -307,7 +307,7 @@ void __init smp_store_boot_cpu_info(void) * The bootstrap kernel entry code has set these up. Save them for * a given CPU */ -void __cpuinit smp_store_cpu_info(int id) +void smp_store_cpu_info(int id) { struct cpuinfo_x86 *c = &cpu_data(id); @@ -320,7 +320,7 @@ void __cpuinit smp_store_cpu_info(int id) identify_secondary_cpu(c); } -static bool __cpuinit +static bool topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name) { int cpu1 = c->cpu_index, cpu2 = o->cpu_index; @@ -337,7 +337,7 @@ do { \ cpumask_set_cpu((c2), cpu_##_m##_mask(c1)); \ } while (0) -static bool __cpuinit match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) +static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) { if (cpu_has_topoext) { int cpu1 = c->cpu_index, cpu2 = o->cpu_index; @@ -355,7 +355,7 @@ static bool __cpuinit match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) return false; } -static bool __cpuinit match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) +static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) { int cpu1 = c->cpu_index, cpu2 = o->cpu_index; @@ -366,7 +366,7 @@ static bool __cpuinit match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) return false; } -static bool __cpuinit match_mc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) +static bool match_mc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) { if (c->phys_proc_id == o->phys_proc_id) { if (cpu_has(c, X86_FEATURE_AMD_DCM)) @@ -377,7 +377,7 @@ static bool __cpuinit match_mc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) return false; } -void __cpuinit set_cpu_sibling_map(int cpu) +void set_cpu_sibling_map(int cpu) { bool has_smt = smp_num_siblings > 1; bool has_mp = has_smt || boot_cpu_data.x86_max_cores > 1; @@ -506,7 +506,7 @@ void __inquire_remote_apic(int apicid) * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this * won't ... remember to clear down the APIC, etc later. */ -int __cpuinit +int wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip) { unsigned long send_status, accept_status = 0; @@ -540,7 +540,7 @@ wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip) return (send_status | accept_status); } -static int __cpuinit +static int wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip) { unsigned long send_status, accept_status = 0; @@ -656,7 +656,7 @@ wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip) } /* reduce the number of lines printed when booting a large cpu count system */ -static void __cpuinit announce_cpu(int cpu, int apicid) +static void announce_cpu(int cpu, int apicid) { static int current_node = -1; int node = early_cpu_to_node(cpu); @@ -698,7 +698,7 @@ static int wakeup_cpu0_nmi(unsigned int cmd, struct pt_regs *regs) * We'll change this code in the future to wake up hard offlined CPU0 if * real platform and request are available. */ -static int __cpuinit +static int wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid, int *cpu0_nmi_registered) { @@ -738,7 +738,7 @@ wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid, * Returns zero if CPU booted OK, else error code from * ->wakeup_secondary_cpu. */ -static int __cpuinit do_boot_cpu(int apicid, int cpu, struct task_struct *idle) +static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle) { volatile u32 *trampoline_status = (volatile u32 *) __va(real_mode_header->trampoline_status); @@ -879,7 +879,7 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu, struct task_struct *idle) return boot_error; } -int __cpuinit native_cpu_up(unsigned int cpu, struct task_struct *tidle) +int native_cpu_up(unsigned int cpu, struct task_struct *tidle) { int apicid = apic->cpu_present_to_apicid(cpu); unsigned long flags; diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c index f84fe00fad48a..68e8057f03c6b 100644 --- a/arch/x86/kernel/tboot.c +++ b/arch/x86/kernel/tboot.c @@ -319,7 +319,7 @@ static int tboot_wait_for_aps(int num_aps) return !(atomic_read((atomic_t *)&tboot->num_in_wfs) == num_aps); } -static int __cpuinit tboot_cpu_callback(struct notifier_block *nfb, +static int tboot_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { switch (action) { @@ -333,7 +333,7 @@ static int __cpuinit tboot_cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block tboot_cpu_notifier __cpuinitdata = +static struct notifier_block tboot_cpu_notifier = { .notifier_call = tboot_cpu_callback, }; diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 9714a7aa32fcd..3dc7633636593 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -827,7 +827,7 @@ static void __init check_system_tsc_reliable(void) * Make an educated guess if the TSC is trustworthy and synchronized * over all CPUs. */ -__cpuinit int unsynchronized_tsc(void) +int unsynchronized_tsc(void) { if (!cpu_has_tsc || tsc_unstable) return 1; @@ -1026,7 +1026,7 @@ void __init tsc_init(void) * been calibrated. This assumes that CONSTANT_TSC applies to all * cpus in the socket - this should be a safe assumption. */ -unsigned long __cpuinit calibrate_delay_is_known(void) +unsigned long calibrate_delay_is_known(void) { int i, cpu = smp_processor_id(); diff --git a/arch/x86/kernel/tsc_sync.c b/arch/x86/kernel/tsc_sync.c index fc25e60a58845..adfdf56a3714e 100644 --- a/arch/x86/kernel/tsc_sync.c +++ b/arch/x86/kernel/tsc_sync.c @@ -25,24 +25,24 @@ * Entry/exit counters that make sure that both CPUs * run the measurement code at once: */ -static __cpuinitdata atomic_t start_count; -static __cpuinitdata atomic_t stop_count; +static atomic_t start_count; +static atomic_t stop_count; /* * We use a raw spinlock in this exceptional case, because * we want to have the fastest, inlined, non-debug version * of a critical section, to be able to prove TSC time-warps: */ -static __cpuinitdata arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED; +static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED; -static __cpuinitdata cycles_t last_tsc; -static __cpuinitdata cycles_t max_warp; -static __cpuinitdata int nr_warps; +static cycles_t last_tsc; +static cycles_t max_warp; +static int nr_warps; /* * TSC-warp measurement loop running on both CPUs: */ -static __cpuinit void check_tsc_warp(unsigned int timeout) +static void check_tsc_warp(unsigned int timeout) { cycles_t start, now, prev, end; int i; @@ -121,7 +121,7 @@ static inline unsigned int loop_timeout(int cpu) * Source CPU calls into this - it waits for the freshly booted * target CPU to arrive and then starts the measurement: */ -void __cpuinit check_tsc_sync_source(int cpu) +void check_tsc_sync_source(int cpu) { int cpus = 2; @@ -187,7 +187,7 @@ void __cpuinit check_tsc_sync_source(int cpu) /* * Freshly booted CPUs call into this: */ -void __cpuinit check_tsc_sync_target(void) +void check_tsc_sync_target(void) { int cpus = 2; diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c index c52c07efe970a..09ce23ae370cc 100644 --- a/arch/x86/kernel/vsyscall_64.c +++ b/arch/x86/kernel/vsyscall_64.c @@ -331,7 +331,7 @@ bool emulate_vsyscall(struct pt_regs *regs, unsigned long address) * Assume __initcall executes before all user space. Hopefully kmod * doesn't violate that. We'll find out if it does. */ -static void __cpuinit vsyscall_set_cpu(int cpu) +static void vsyscall_set_cpu(int cpu) { unsigned long d; unsigned long node = 0; @@ -353,13 +353,13 @@ static void __cpuinit vsyscall_set_cpu(int cpu) write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S); } -static void __cpuinit cpu_vsyscall_init(void *arg) +static void cpu_vsyscall_init(void *arg) { /* preemption should be already off */ vsyscall_set_cpu(raw_smp_processor_id()); } -static int __cpuinit +static int cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg) { long cpu = (long)arg; diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c index 45a14dbbddafe..5f24c71accaaf 100644 --- a/arch/x86/kernel/x86_init.c +++ b/arch/x86/kernel/x86_init.c @@ -25,7 +25,7 @@ #include #include -void __cpuinit x86_init_noop(void) { } +void x86_init_noop(void) { } void __init x86_init_uint_noop(unsigned int unused) { } int __init iommu_init_noop(void) { return 0; } void iommu_shutdown_noop(void) { } @@ -85,7 +85,7 @@ struct x86_init_ops x86_init __initdata = { }, }; -struct x86_cpuinit_ops x86_cpuinit __cpuinitdata = { +struct x86_cpuinit_ops x86_cpuinit = { .early_percpu_clock_init = x86_init_noop, .setup_percpu_clockev = setup_secondary_APIC_clock, }; diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c index 92f37e7683c50..7ed034752e934 100644 --- a/arch/x86/kernel/xsave.c +++ b/arch/x86/kernel/xsave.c @@ -576,7 +576,7 @@ static void __init xstate_enable_boot_cpu(void) * This is somewhat obfuscated due to the lack of powerful enough * overrides for the section checks. */ -void __cpuinit xsave_init(void) +void xsave_init(void) { static __refdata void (*next_func)(void) = xstate_enable_boot_cpu; void (*this_func)(void); @@ -597,7 +597,7 @@ static inline void __init eager_fpu_init_bp(void) setup_init_fpu_buf(); } -void __cpuinit eager_fpu_init(void) +void eager_fpu_init(void) { static __refdata void (*boot_func)(void) = eager_fpu_init_bp; diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c index a71c4e207679a..8bf93bae1f13f 100644 --- a/arch/x86/mm/numa.c +++ b/arch/x86/mm/numa.c @@ -60,7 +60,7 @@ s16 __apicid_to_node[MAX_LOCAL_APIC] = { [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE }; -int __cpuinit numa_cpu_node(int cpu) +int numa_cpu_node(int cpu) { int apicid = early_per_cpu(x86_cpu_to_apicid, cpu); @@ -691,12 +691,12 @@ void __init init_cpu_to_node(void) #ifndef CONFIG_DEBUG_PER_CPU_MAPS # ifndef CONFIG_NUMA_EMU -void __cpuinit numa_add_cpu(int cpu) +void numa_add_cpu(int cpu) { cpumask_set_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]); } -void __cpuinit numa_remove_cpu(int cpu) +void numa_remove_cpu(int cpu) { cpumask_clear_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]); } @@ -763,17 +763,17 @@ void debug_cpumask_set_cpu(int cpu, int node, bool enable) } # ifndef CONFIG_NUMA_EMU -static void __cpuinit numa_set_cpumask(int cpu, bool enable) +static void numa_set_cpumask(int cpu, bool enable) { debug_cpumask_set_cpu(cpu, early_cpu_to_node(cpu), enable); } -void __cpuinit numa_add_cpu(int cpu) +void numa_add_cpu(int cpu) { numa_set_cpumask(cpu, true); } -void __cpuinit numa_remove_cpu(int cpu) +void numa_remove_cpu(int cpu) { numa_set_cpumask(cpu, false); } diff --git a/arch/x86/mm/numa_emulation.c b/arch/x86/mm/numa_emulation.c index dbbbb47260ccc..67c2d21b800b9 100644 --- a/arch/x86/mm/numa_emulation.c +++ b/arch/x86/mm/numa_emulation.c @@ -444,7 +444,7 @@ void __init numa_emulation(struct numa_meminfo *numa_meminfo, int numa_dist_cnt) } #ifndef CONFIG_DEBUG_PER_CPU_MAPS -void __cpuinit numa_add_cpu(int cpu) +void numa_add_cpu(int cpu) { int physnid, nid; @@ -462,7 +462,7 @@ void __cpuinit numa_add_cpu(int cpu) cpumask_set_cpu(cpu, node_to_cpumask_map[nid]); } -void __cpuinit numa_remove_cpu(int cpu) +void numa_remove_cpu(int cpu) { int i; @@ -470,7 +470,7 @@ void __cpuinit numa_remove_cpu(int cpu) cpumask_clear_cpu(cpu, node_to_cpumask_map[i]); } #else /* !CONFIG_DEBUG_PER_CPU_MAPS */ -static void __cpuinit numa_set_cpumask(int cpu, bool enable) +static void numa_set_cpumask(int cpu, bool enable) { int nid, physnid; @@ -490,12 +490,12 @@ static void __cpuinit numa_set_cpumask(int cpu, bool enable) } } -void __cpuinit numa_add_cpu(int cpu) +void numa_add_cpu(int cpu) { numa_set_cpumask(cpu, true); } -void __cpuinit numa_remove_cpu(int cpu) +void numa_remove_cpu(int cpu) { numa_set_cpumask(cpu, false); } diff --git a/arch/x86/mm/setup_nx.c b/arch/x86/mm/setup_nx.c index 410531d3c292d..da23af2868821 100644 --- a/arch/x86/mm/setup_nx.c +++ b/arch/x86/mm/setup_nx.c @@ -29,7 +29,7 @@ static int __init noexec_setup(char *str) } early_param("noexec", noexec_setup); -void __cpuinit x86_configure_nx(void) +void x86_configure_nx(void) { if (cpu_has_nx && !disable_nx) __supported_pte_mask |= _PAGE_NX; diff --git a/arch/x86/pci/amd_bus.c b/arch/x86/pci/amd_bus.c index e9e6ed5cdf946..a0077b78ae86c 100644 --- a/arch/x86/pci/amd_bus.c +++ b/arch/x86/pci/amd_bus.c @@ -312,7 +312,7 @@ static int __init early_fill_mp_bus_info(void) #define ENABLE_CF8_EXT_CFG (1ULL << 46) -static void __cpuinit enable_pci_io_ecs(void *unused) +static void enable_pci_io_ecs(void *unused) { u64 reg; rdmsrl(MSR_AMD64_NB_CFG, reg); @@ -322,7 +322,7 @@ static void __cpuinit enable_pci_io_ecs(void *unused) } } -static int __cpuinit amd_cpu_notify(struct notifier_block *self, +static int amd_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { int cpu = (long)hcpu; @@ -337,7 +337,7 @@ static int __cpuinit amd_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata amd_cpu_notifier = { +static struct notifier_block amd_cpu_notifier = { .notifier_call = amd_cpu_notify, }; diff --git a/arch/x86/platform/ce4100/ce4100.c b/arch/x86/platform/ce4100/ce4100.c index f8ab4945892ee..baec704231b31 100644 --- a/arch/x86/platform/ce4100/ce4100.c +++ b/arch/x86/platform/ce4100/ce4100.c @@ -134,7 +134,7 @@ static void __init sdv_arch_setup(void) } #ifdef CONFIG_X86_IO_APIC -static void __cpuinit sdv_pci_init(void) +static void sdv_pci_init(void) { x86_of_pci_init(); /* We can't set this earlier, because we need to calibrate the timer */ diff --git a/arch/x86/platform/mrst/mrst.c b/arch/x86/platform/mrst/mrst.c index a0a0a4389bbd9..47fe66fe61f12 100644 --- a/arch/x86/platform/mrst/mrst.c +++ b/arch/x86/platform/mrst/mrst.c @@ -65,7 +65,7 @@ * lapic (always-on,ARAT) ------ 150 */ -__cpuinitdata enum mrst_timer_options mrst_timer_options; +enum mrst_timer_options mrst_timer_options; static u32 sfi_mtimer_usage[SFI_MTMR_MAX_NUM]; static struct sfi_timer_table_entry sfi_mtimer_array[SFI_MTMR_MAX_NUM]; @@ -248,7 +248,7 @@ static void __init mrst_time_init(void) apbt_time_init(); } -static void __cpuinit mrst_arch_setup(void) +static void mrst_arch_setup(void) { if (boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model == 0x27) __mrst_cpu_chip = MRST_CPU_CHIP_PENWELL; diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 34511cf6baad6..a1531aa0d845f 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -1725,7 +1725,7 @@ static void __init init_hvm_pv_info(void) xen_domain_type = XEN_HVM_DOMAIN; } -static int __cpuinit xen_hvm_cpu_notify(struct notifier_block *self, +static int xen_hvm_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { int cpu = (long)hcpu; @@ -1744,7 +1744,7 @@ static int __cpuinit xen_hvm_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block xen_hvm_cpu_notifier __cpuinitdata = { +static struct notifier_block xen_hvm_cpu_notifier = { .notifier_call = xen_hvm_cpu_notify, }; diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c index 0a9fb7a0b4521..8f3eea6b80c52 100644 --- a/arch/x86/xen/setup.c +++ b/arch/x86/xen/setup.c @@ -497,7 +497,7 @@ static void __init fiddle_vdso(void) #endif } -static int __cpuinit register_callback(unsigned type, const void *func) +static int register_callback(unsigned type, const void *func) { struct callback_register callback = { .type = type, @@ -508,7 +508,7 @@ static int __cpuinit register_callback(unsigned type, const void *func) return HYPERVISOR_callback_op(CALLBACKOP_register, &callback); } -void __cpuinit xen_enable_sysenter(void) +void xen_enable_sysenter(void) { int ret; unsigned sysenter_feature; @@ -527,7 +527,7 @@ void __cpuinit xen_enable_sysenter(void) setup_clear_cpu_cap(sysenter_feature); } -void __cpuinit xen_enable_syscall(void) +void xen_enable_syscall(void) { #ifdef CONFIG_X86_64 int ret; diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c index 570c9a5c4d3f6..bee9e9b9cb4c0 100644 --- a/arch/x86/xen/smp.c +++ b/arch/x86/xen/smp.c @@ -61,7 +61,7 @@ static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static void __cpuinit cpu_bringup(void) +static void cpu_bringup(void) { int cpu; @@ -93,7 +93,7 @@ static void __cpuinit cpu_bringup(void) wmb(); /* make sure everything is out */ } -static void __cpuinit cpu_bringup_and_idle(void) +static void cpu_bringup_and_idle(void) { cpu_bringup(); cpu_startup_entry(CPUHP_ONLINE); @@ -303,7 +303,7 @@ static void __init xen_smp_prepare_cpus(unsigned int max_cpus) set_cpu_present(cpu, true); } -static int __cpuinit +static int cpu_initialize_context(unsigned int cpu, struct task_struct *idle) { struct vcpu_guest_context *ctxt; @@ -374,7 +374,7 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle) return 0; } -static int __cpuinit xen_cpu_up(unsigned int cpu, struct task_struct *idle) +static int xen_cpu_up(unsigned int cpu, struct task_struct *idle) { int rc; @@ -452,7 +452,7 @@ static void xen_cpu_die(unsigned int cpu) xen_teardown_timer(cpu); } -static void __cpuinit xen_play_dead(void) /* used only with HOTPLUG_CPU */ +static void xen_play_dead(void) /* used only with HOTPLUG_CPU */ { play_dead_common(); HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL); @@ -673,7 +673,7 @@ static void __init xen_hvm_smp_prepare_cpus(unsigned int max_cpus) xen_init_lock_cpu(0); } -static int __cpuinit xen_hvm_cpu_up(unsigned int cpu, struct task_struct *tidle) +static int xen_hvm_cpu_up(unsigned int cpu, struct task_struct *tidle) { int rc; /* diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c index 3002ec1bb71a2..9553b2b917363 100644 --- a/arch/x86/xen/spinlock.c +++ b/arch/x86/xen/spinlock.c @@ -359,7 +359,7 @@ static irqreturn_t dummy_handler(int irq, void *dev_id) return IRQ_HANDLED; } -void __cpuinit xen_init_lock_cpu(int cpu) +void xen_init_lock_cpu(int cpu) { int irq; const char *name; diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index a95b41744ad0b..86782c5d7e2a6 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h @@ -73,7 +73,7 @@ static inline void xen_hvm_smp_init(void) {} #ifdef CONFIG_PARAVIRT_SPINLOCKS void __init xen_init_spinlocks(void); -void __cpuinit xen_init_lock_cpu(int cpu); +void xen_init_lock_cpu(int cpu); void xen_uninit_lock_cpu(int cpu); #else static inline void xen_init_spinlocks(void) diff --git a/arch/xtensa/kernel/time.c b/arch/xtensa/kernel/time.c index ffb4741043116..f1818db1d9882 100644 --- a/arch/xtensa/kernel/time.c +++ b/arch/xtensa/kernel/time.c @@ -109,7 +109,7 @@ irqreturn_t timer_interrupt (int irq, void *dev_id) } #ifndef CONFIG_GENERIC_CALIBRATE_DELAY -void __cpuinit calibrate_delay(void) +void calibrate_delay(void) { loops_per_jiffy = CCOUNT_PER_JIFFY; printk("Calibrating delay loop (skipped)... " diff --git a/block/blk-iopoll.c b/block/blk-iopoll.c index 2d4d8d5674412..5b9432264e69a 100644 --- a/block/blk-iopoll.c +++ b/block/blk-iopoll.c @@ -189,7 +189,7 @@ void blk_iopoll_init(struct blk_iopoll *iop, int weight, blk_iopoll_fn *poll_fn) } EXPORT_SYMBOL(blk_iopoll_init); -static int __cpuinit blk_iopoll_cpu_notify(struct notifier_block *self, +static int blk_iopoll_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { /* @@ -209,7 +209,7 @@ static int __cpuinit blk_iopoll_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata blk_iopoll_cpu_notifier = { +static struct notifier_block blk_iopoll_cpu_notifier = { .notifier_call = blk_iopoll_cpu_notify, }; diff --git a/block/blk-softirq.c b/block/blk-softirq.c index 2b01c0b739848..76b9d2f7cc133 100644 --- a/block/blk-softirq.c +++ b/block/blk-softirq.c @@ -78,7 +78,7 @@ static int raise_blk_irq(int cpu, struct request *rq) } #endif -static int __cpuinit blk_cpu_notify(struct notifier_block *self, +static int blk_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { /* @@ -98,7 +98,7 @@ static int __cpuinit blk_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata blk_cpu_notifier = { +static struct notifier_block blk_cpu_notifier = { .notifier_call = blk_cpu_notify, }; diff --git a/drivers/acpi/processor_core.c b/drivers/acpi/processor_core.c index 164d49569aebf..a5e9f4a5b2816 100644 --- a/drivers/acpi/processor_core.c +++ b/drivers/acpi/processor_core.c @@ -253,7 +253,7 @@ static bool __init processor_physically_present(acpi_handle handle) return true; } -static void __cpuinit acpi_set_pdc_bits(u32 *buf) +static void acpi_set_pdc_bits(u32 *buf) { buf[0] = ACPI_PDC_REVISION_ID; buf[1] = 1; @@ -265,7 +265,7 @@ static void __cpuinit acpi_set_pdc_bits(u32 *buf) arch_acpi_set_pdc_bits(buf); } -static struct acpi_object_list *__cpuinit acpi_processor_alloc_pdc(void) +static struct acpi_object_list *acpi_processor_alloc_pdc(void) { struct acpi_object_list *obj_list; union acpi_object *obj; @@ -308,7 +308,7 @@ static struct acpi_object_list *__cpuinit acpi_processor_alloc_pdc(void) * _PDC is required for a BIOS-OS handshake for most of the newer * ACPI processor features. */ -static int __cpuinit +static int acpi_processor_eval_pdc(acpi_handle handle, struct acpi_object_list *pdc_in) { acpi_status status = AE_OK; @@ -336,7 +336,7 @@ acpi_processor_eval_pdc(acpi_handle handle, struct acpi_object_list *pdc_in) return status; } -void __cpuinit acpi_processor_set_pdc(acpi_handle handle) +void acpi_processor_set_pdc(acpi_handle handle) { struct acpi_object_list *obj_list; diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c index c266cdc117840..3efb90bb0f90c 100644 --- a/drivers/acpi/processor_driver.c +++ b/drivers/acpi/processor_driver.c @@ -461,7 +461,7 @@ static struct notifier_block acpi_cpu_notifier = * acpi_cpu_soft_notify(). Getting it __cpuinit{data} is difficult, the * root cause seem to be that acpi_processor_uninstall_hotplug_notify() * is in the module_exit (__exit) func. Allowing acpi_processor_start() - * to not be in __cpuinit section, but being called from __cpuinit funcs + * to not be in section, but being called from funcs * via __ref looks like the right thing to do here. */ static __ref int acpi_processor_start(struct acpi_processor *pr) @@ -524,7 +524,7 @@ static __ref int acpi_processor_start(struct acpi_processor *pr) * (cpu_data(cpu)) values, like CPU feature flags, family, model, etc. * Such things have to be put in and set up above in acpi_processor_start() */ -static int __cpuinit acpi_processor_add(struct acpi_device *device) +static int acpi_processor_add(struct acpi_device *device) { struct acpi_processor *pr = NULL; int result = 0; diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index c991fe680e589..1b8ade5ad3801 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -98,7 +98,7 @@ static int set_max_cstate(const struct dmi_system_id *id) /* Actually this shouldn't be __cpuinitdata, would be better to fix the callers to only run once -AK */ -static struct dmi_system_id __cpuinitdata processor_power_dmi_table[] = { +static struct dmi_system_id processor_power_dmi_table[] = { { set_max_cstate, "Clevo 5600D", { DMI_MATCH(DMI_BIOS_VENDOR,"Phoenix Technologies LTD"), DMI_MATCH(DMI_BIOS_VERSION,"SHE845M0.86C.0013.D.0302131307")}, @@ -1139,7 +1139,7 @@ int acpi_processor_cst_has_changed(struct acpi_processor *pr) static int acpi_processor_registered; -int __cpuinit acpi_processor_power_init(struct acpi_processor *pr) +int acpi_processor_power_init(struct acpi_processor *pr) { acpi_status status = 0; int retval; diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index d5963ed5e5b13..821b91af9207c 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -87,7 +87,7 @@ static ssize_t __ref store_online(struct device *dev, } static DEVICE_ATTR(online, 0644, show_online, store_online); -static void __cpuinit register_cpu_control(struct cpu *cpu) +static void register_cpu_control(struct cpu *cpu) { device_create_file(&cpu->dev, &dev_attr_online); } @@ -465,7 +465,7 @@ static int cpu_uevent(struct device *dev, struct kobj_uevent_env *env) * * Initialize and register the CPU device. */ -int __cpuinit register_cpu(struct cpu *cpu, int num) +int register_cpu(struct cpu *cpu, int num) { int error; diff --git a/drivers/base/topology.c b/drivers/base/topology.c index 1eeb5f0048810..e4306a834b2e1 100644 --- a/drivers/base/topology.c +++ b/drivers/base/topology.c @@ -142,21 +142,21 @@ static struct attribute_group topology_attr_group = { }; /* Add/Remove cpu_topology interface for CPU device */ -static int __cpuinit topology_add_dev(unsigned int cpu) +static int topology_add_dev(unsigned int cpu) { struct device *dev = get_cpu_device(cpu); return sysfs_create_group(&dev->kobj, &topology_attr_group); } -static void __cpuinit topology_remove_dev(unsigned int cpu) +static void topology_remove_dev(unsigned int cpu) { struct device *dev = get_cpu_device(cpu); sysfs_remove_group(&dev->kobj, &topology_attr_group); } -static int __cpuinit topology_cpu_callback(struct notifier_block *nfb, +static int topology_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; @@ -177,7 +177,7 @@ static int __cpuinit topology_cpu_callback(struct notifier_block *nfb, return notifier_from_errno(rc); } -static int __cpuinit topology_sysfs_init(void) +static int topology_sysfs_init(void) { int cpu; int rc = 0; diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 1bdf38aab486f..dccb7e0c97e11 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -268,7 +268,7 @@ static int arch_timer_set_next_event_phys_mem(unsigned long evt, return 0; } -static void __cpuinit __arch_timer_setup(unsigned type, +static void __arch_timer_setup(unsigned type, struct clock_event_device *clk) { clk->features = CLOCK_EVT_FEAT_ONESHOT; @@ -499,7 +499,7 @@ static void __init arch_counter_register(unsigned type) sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate); } -static void __cpuinit arch_timer_stop(struct clock_event_device *clk) +static void arch_timer_stop(struct clock_event_device *clk) { pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id()); @@ -515,7 +515,7 @@ static void __cpuinit arch_timer_stop(struct clock_event_device *clk) clk->set_mode(CLOCK_EVT_MODE_UNUSED, clk); } -static int __cpuinit arch_timer_cpu_notify(struct notifier_block *self, +static int arch_timer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { /* @@ -534,7 +534,7 @@ static int __cpuinit arch_timer_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block arch_timer_cpu_nb __cpuinitdata = { +static struct notifier_block arch_timer_cpu_nb = { .notifier_call = arch_timer_cpu_notify, }; diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c index 3dfa3e5e3705e..7b98e6472631e 100644 --- a/drivers/clocksource/exynos_mct.c +++ b/drivers/clocksource/exynos_mct.c @@ -412,7 +412,7 @@ static struct irqaction mct_tick1_event_irq = { .handler = exynos4_mct_tick_isr, }; -static int __cpuinit exynos4_local_timer_setup(struct clock_event_device *evt) +static int exynos4_local_timer_setup(struct clock_event_device *evt) { struct mct_clock_event_device *mevt; unsigned int cpu = smp_processor_id(); @@ -465,7 +465,7 @@ static void exynos4_local_timer_stop(struct clock_event_device *evt) disable_percpu_irq(mct_irqs[MCT_L0_IRQ]); } -static struct local_timer_ops exynos4_mct_tick_ops __cpuinitdata = { +static struct local_timer_ops exynos4_mct_tick_ops = { .setup = exynos4_local_timer_setup, .stop = exynos4_local_timer_stop, }; diff --git a/drivers/clocksource/metag_generic.c b/drivers/clocksource/metag_generic.c index ade7513a11d1d..05ccb99acf3ef 100644 --- a/drivers/clocksource/metag_generic.c +++ b/drivers/clocksource/metag_generic.c @@ -109,7 +109,7 @@ unsigned long long sched_clock(void) return ticks << HARDWARE_TO_NS_SHIFT; } -static void __cpuinit arch_timer_setup(unsigned int cpu) +static void arch_timer_setup(unsigned int cpu) { unsigned int txdivtime; struct clock_event_device *clk = &per_cpu(local_clockevent, cpu); @@ -154,7 +154,7 @@ static void __cpuinit arch_timer_setup(unsigned int cpu) } } -static int __cpuinit arch_timer_cpu_notify(struct notifier_block *self, +static int arch_timer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { int cpu = (long)hcpu; @@ -169,7 +169,7 @@ static int __cpuinit arch_timer_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata arch_timer_cpu_nb = { +static struct notifier_block arch_timer_cpu_nb = { .notifier_call = arch_timer_cpu_notify, }; diff --git a/drivers/clocksource/time-armada-370-xp.c b/drivers/clocksource/time-armada-370-xp.c index efdca3263afef..1b04b7e1d39b9 100644 --- a/drivers/clocksource/time-armada-370-xp.c +++ b/drivers/clocksource/time-armada-370-xp.c @@ -167,7 +167,7 @@ static irqreturn_t armada_370_xp_timer_interrupt(int irq, void *dev_id) /* * Setup the local clock events for a CPU. */ -static int __cpuinit armada_370_xp_timer_setup(struct clock_event_device *evt) +static int armada_370_xp_timer_setup(struct clock_event_device *evt) { u32 u; int cpu = smp_processor_id(); @@ -205,7 +205,7 @@ static void armada_370_xp_timer_stop(struct clock_event_device *evt) disable_percpu_irq(evt->irq); } -static struct local_timer_ops armada_370_xp_local_timer_ops __cpuinitdata = { +static struct local_timer_ops armada_370_xp_local_timer_ops = { .setup = armada_370_xp_timer_setup, .stop = armada_370_xp_timer_stop, }; diff --git a/drivers/clocksource/timer-marco.c b/drivers/clocksource/timer-marco.c index e5dc9129ca263..62876baa3ab9c 100644 --- a/drivers/clocksource/timer-marco.c +++ b/drivers/clocksource/timer-marco.c @@ -184,7 +184,7 @@ static struct irqaction sirfsoc_timer1_irq = { .handler = sirfsoc_timer_interrupt, }; -static int __cpuinit sirfsoc_local_timer_setup(struct clock_event_device *ce) +static int sirfsoc_local_timer_setup(struct clock_event_device *ce) { /* Use existing clock_event for cpu 0 */ if (!smp_processor_id()) @@ -216,7 +216,7 @@ static void sirfsoc_local_timer_stop(struct clock_event_device *ce) remove_irq(sirfsoc_timer1_irq.irq, &sirfsoc_timer1_irq); } -static struct local_timer_ops sirfsoc_local_timer_ops __cpuinitdata = { +static struct local_timer_ops sirfsoc_local_timer_ops = { .setup = sirfsoc_local_timer_setup, .stop = sirfsoc_local_timer_stop, }; diff --git a/drivers/cpufreq/cpufreq_persistent_stats.c b/drivers/cpufreq/cpufreq_persistent_stats.c index 0b2ded84e17b8..dbd86f30a49f1 100644 --- a/drivers/cpufreq/cpufreq_persistent_stats.c +++ b/drivers/cpufreq/cpufreq_persistent_stats.c @@ -268,7 +268,7 @@ static int cpufreq_stat_notifier_trans(struct notifier_block *nb, return 0; } -static int __cpuinit cpufreq_stat_cpu_callback(struct notifier_block *nfb, +static int cpufreq_stat_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index 0cf25101b3c3e..9f8ac97e78be9 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -195,7 +195,7 @@ struct tjmax { int tjmax; }; -static const struct tjmax __cpuinitconst tjmax_table[] = { +static const struct tjmax tjmax_table[] = { { "CPU 230", 100000 }, /* Model 0x1c, stepping 2 */ { "CPU 330", 125000 }, /* Model 0x1c, stepping 2 */ { "CPU CE4110", 110000 }, /* Model 0x1c, stepping 10 Sodaville */ @@ -211,7 +211,7 @@ struct tjmax_model { #define ANY 0xff -static const struct tjmax_model __cpuinitconst tjmax_model_table[] = { +static const struct tjmax_model tjmax_model_table[] = { { 0x1c, 10, 100000 }, /* D4xx, K4xx, N4xx, D5xx, K5xx, N5xx */ { 0x1c, ANY, 90000 }, /* Z5xx, N2xx, possibly others * Note: Also matches 230 and 330, @@ -226,7 +226,7 @@ static const struct tjmax_model __cpuinitconst tjmax_model_table[] = { { 0x36, ANY, 100000 }, /* Atom Cedar Trail/Cedarview (N2xxx, D2xxx) */ }; -static int __cpuinit adjust_tjmax(struct cpuinfo_x86 *c, u32 id, +static int adjust_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev) { /* The 100C is default for both mobile and non mobile CPUs */ @@ -317,7 +317,7 @@ static int __cpuinit adjust_tjmax(struct cpuinfo_x86 *c, u32 id, return tjmax; } -static int __cpuinit get_tjmax(struct cpuinfo_x86 *c, u32 id, +static int get_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev) { int err; @@ -367,7 +367,7 @@ static int create_name_attr(struct platform_data *pdata, return device_create_file(dev, &pdata->name_attr); } -static int __cpuinit create_core_attrs(struct temp_data *tdata, +static int create_core_attrs(struct temp_data *tdata, struct device *dev, int attr_no) { int err, i; @@ -401,7 +401,7 @@ static int __cpuinit create_core_attrs(struct temp_data *tdata, } -static int __cpuinit chk_ucode_version(unsigned int cpu) +static int chk_ucode_version(unsigned int cpu) { struct cpuinfo_x86 *c = &cpu_data(cpu); @@ -417,7 +417,7 @@ static int __cpuinit chk_ucode_version(unsigned int cpu) return 0; } -static struct platform_device __cpuinit *coretemp_get_pdev(unsigned int cpu) +static struct platform_device *coretemp_get_pdev(unsigned int cpu) { u16 phys_proc_id = TO_PHYS_ID(cpu); struct pdev_entry *p; @@ -434,7 +434,7 @@ static struct platform_device __cpuinit *coretemp_get_pdev(unsigned int cpu) return NULL; } -static struct temp_data __cpuinit *init_temp_data(unsigned int cpu, +static struct temp_data *init_temp_data(unsigned int cpu, int pkg_flag) { struct temp_data *tdata; @@ -453,7 +453,7 @@ static struct temp_data __cpuinit *init_temp_data(unsigned int cpu, return tdata; } -static int __cpuinit create_core_data(struct platform_device *pdev, +static int create_core_data(struct platform_device *pdev, unsigned int cpu, int pkg_flag) { struct temp_data *tdata; @@ -524,7 +524,7 @@ static int __cpuinit create_core_data(struct platform_device *pdev, return err; } -static void __cpuinit coretemp_add_core(unsigned int cpu, int pkg_flag) +static void coretemp_add_core(unsigned int cpu, int pkg_flag) { struct platform_device *pdev = coretemp_get_pdev(cpu); int err; @@ -609,7 +609,7 @@ static struct platform_driver coretemp_driver = { .remove = coretemp_remove, }; -static int __cpuinit coretemp_device_add(unsigned int cpu) +static int coretemp_device_add(unsigned int cpu) { int err; struct platform_device *pdev; @@ -653,7 +653,7 @@ static int __cpuinit coretemp_device_add(unsigned int cpu) return err; } -static void __cpuinit coretemp_device_remove(unsigned int cpu) +static void coretemp_device_remove(unsigned int cpu) { struct pdev_entry *p, *n; u16 phys_proc_id = TO_PHYS_ID(cpu); @@ -669,7 +669,7 @@ static void __cpuinit coretemp_device_remove(unsigned int cpu) mutex_unlock(&pdev_list_mutex); } -static bool __cpuinit is_any_core_online(struct platform_data *pdata) +static bool is_any_core_online(struct platform_data *pdata) { int i; @@ -683,7 +683,7 @@ static bool __cpuinit is_any_core_online(struct platform_data *pdata) return false; } -static void __cpuinit get_core_online(unsigned int cpu) +static void get_core_online(unsigned int cpu) { struct cpuinfo_x86 *c = &cpu_data(cpu); struct platform_device *pdev = coretemp_get_pdev(cpu); @@ -725,7 +725,7 @@ static void __cpuinit get_core_online(unsigned int cpu) coretemp_add_core(cpu, 0); } -static void __cpuinit put_core_offline(unsigned int cpu) +static void put_core_offline(unsigned int cpu) { int i, indx; struct platform_data *pdata; @@ -773,7 +773,7 @@ static void __cpuinit put_core_offline(unsigned int cpu) coretemp_device_remove(cpu); } -static int __cpuinit coretemp_cpu_callback(struct notifier_block *nfb, +static int coretemp_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long) hcpu; diff --git a/drivers/hwmon/via-cputemp.c b/drivers/hwmon/via-cputemp.c index 76f157b568ed7..65edf7da86edb 100644 --- a/drivers/hwmon/via-cputemp.c +++ b/drivers/hwmon/via-cputemp.c @@ -221,7 +221,7 @@ struct pdev_entry { static LIST_HEAD(pdev_list); static DEFINE_MUTEX(pdev_list_mutex); -static int __cpuinit via_cputemp_device_add(unsigned int cpu) +static int via_cputemp_device_add(unsigned int cpu) { int err; struct platform_device *pdev; @@ -262,7 +262,7 @@ static int __cpuinit via_cputemp_device_add(unsigned int cpu) return err; } -static void __cpuinit via_cputemp_device_remove(unsigned int cpu) +static void via_cputemp_device_remove(unsigned int cpu) { struct pdev_entry *p; @@ -279,7 +279,7 @@ static void __cpuinit via_cputemp_device_remove(unsigned int cpu) mutex_unlock(&pdev_list_mutex); } -static int __cpuinit via_cputemp_cpu_callback(struct notifier_block *nfb, +static int via_cputemp_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long) hcpu; diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index f26ff9e9a55fd..e5ca0fbd41028 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -630,7 +630,7 @@ static void __init gic_dist_init(struct gic_chip_data *gic) mb(); } -static void __cpuinit gic_cpu_init(struct gic_chip_data *gic) +static void gic_cpu_init(struct gic_chip_data *gic) { void __iomem *dist_base = gic_data_dist_base(gic); void __iomem *base = gic_data_cpu_base(gic); @@ -990,7 +990,7 @@ static int gic_irq_domain_xlate(struct irq_domain *d, } #ifdef CONFIG_SMP -static int __cpuinit gic_secondary_init(struct notifier_block *nfb, +static int gic_secondary_init(struct notifier_block *nfb, unsigned long action, void *hcpu) { if (action == CPU_STARTING || action == CPU_STARTING_FROZEN) @@ -1002,7 +1002,7 @@ static int __cpuinit gic_secondary_init(struct notifier_block *nfb, * Notifier for enabling the GIC CPU interface. Set an arbitrarily high * priority because the GIC needs to be up before the ARM generic timers. */ -static struct notifier_block __cpuinitdata gic_cpu_notifier = { +static struct notifier_block gic_cpu_notifier = { .notifier_call = gic_secondary_init, .priority = 100, }; diff --git a/drivers/oprofile/timer_int.c b/drivers/oprofile/timer_int.c index 93404f72dfa81..235659af2c425 100644 --- a/drivers/oprofile/timer_int.c +++ b/drivers/oprofile/timer_int.c @@ -74,7 +74,7 @@ static void oprofile_hrtimer_stop(void) put_online_cpus(); } -static int __cpuinit oprofile_cpu_notify(struct notifier_block *self, +static int oprofile_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { long cpu = (long) hcpu; diff --git a/drivers/xen/xen-acpi-cpuhotplug.c b/drivers/xen/xen-acpi-cpuhotplug.c index 18c742bec91bf..b7d5d95d8abf9 100644 --- a/drivers/xen/xen-acpi-cpuhotplug.c +++ b/drivers/xen/xen-acpi-cpuhotplug.c @@ -89,7 +89,7 @@ static int xen_acpi_processor_enable(struct acpi_device *device) return 0; } -static int __cpuinit xen_acpi_processor_add(struct acpi_device *device) +static int xen_acpi_processor_add(struct acpi_device *device) { int ret; struct acpi_processor *pr; diff --git a/include/linux/cpu.h b/include/linux/cpu.h index 33f55c8c59d1f..70084db409f68 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -129,13 +129,13 @@ enum { /* Need to know about CPUs going up/down? */ #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) #define cpu_notifier(fn, pri) { \ - static struct notifier_block fn##_nb __cpuinitdata = \ + static struct notifier_block fn##_nb = \ { .notifier_call = fn, .priority = pri }; \ register_cpu_notifier(&fn##_nb); \ } #define __cpu_notifier(fn, pri) { \ - static struct notifier_block fn##_nb __cpuinitdata = \ + static struct notifier_block fn##_nb = \ { .notifier_call = fn, .priority = pri }; \ __register_cpu_notifier(&fn##_nb); \ } diff --git a/include/linux/init.h b/include/linux/init.h index 861814710d523..20d992b73d72e 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -94,12 +94,12 @@ #define __exit __section(.exit.text) __exitused __cold notrace /* Used for HOTPLUG_CPU */ -#define __cpuinit __section(.cpuinit.text) __cold notrace -#define __cpuinitdata __section(.cpuinit.data) -#define __cpuinitconst __constsection(.cpuinit.rodata) -#define __cpuexit __section(.cpuexit.text) __exitused __cold notrace -#define __cpuexitdata __section(.cpuexit.data) -#define __cpuexitconst __constsection(.cpuexit.rodata) +#define __cpuinit __cold +#define __cpuinitdata +#define __cpuinitconst +#define __cpuexit __exitused __cold +#define __cpuexitdata +#define __cpuexitconst /* Used for MEMORY_HOTPLUG */ #define __meminit __section(.meminit.text) __cold notrace diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index b10868f51332e..54aad39a8a82e 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -826,7 +826,7 @@ static inline void perf_restore_debug_store(void) { } */ #define perf_cpu_notifier(fn) \ do { \ - static struct notifier_block fn##_nb __cpuinitdata = \ + static struct notifier_block fn##_nb = \ { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ unsigned long cpu = smp_processor_id(); \ unsigned long flags; \ diff --git a/init/calibrate.c b/init/calibrate.c index fda0a7b0f06c0..fe85c605c7d7b 100644 --- a/init/calibrate.c +++ b/init/calibrate.c @@ -31,7 +31,7 @@ __setup("lpj=", lpj_setup); #define DELAY_CALIBRATION_TICKS ((HZ < 100) ? 1 : (HZ/100)) #define MAX_DIRECT_CALIBRATION_RETRIES 5 -static unsigned long __cpuinit calibrate_delay_direct(void) +static unsigned long calibrate_delay_direct(void) { unsigned long pre_start, start, post_start; unsigned long pre_end, end, post_end; @@ -166,7 +166,7 @@ static unsigned long __cpuinit calibrate_delay_direct(void) return 0; } #else -static unsigned long __cpuinit calibrate_delay_direct(void) {return 0;} +static unsigned long calibrate_delay_direct(void) {return 0;} #endif /* @@ -180,7 +180,7 @@ static unsigned long __cpuinit calibrate_delay_direct(void) {return 0;} */ #define LPS_PREC 8 -static unsigned long __cpuinit calibrate_delay_converge(void) +static unsigned long calibrate_delay_converge(void) { /* First stage - slowly accelerate to find initial bounds */ unsigned long lpj, lpj_base, ticks, loopadd, loopadd_base, chop_limit; @@ -254,12 +254,12 @@ static DEFINE_PER_CPU(unsigned long, cpu_loops_per_jiffy) = { 0 }; * Architectures should override this function if a faster calibration * method is available. */ -unsigned long __attribute__((weak)) __cpuinit calibrate_delay_is_known(void) +unsigned long __attribute__((weak)) calibrate_delay_is_known(void) { return 0; } -void __cpuinit calibrate_delay(void) +void calibrate_delay(void) { unsigned long lpj; static bool printed; diff --git a/kernel/cpu.c b/kernel/cpu.c index 5ac6ff97a98b2..575906b0ea03c 100755 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -384,7 +384,7 @@ EXPORT_SYMBOL(cpu_down); #endif /*CONFIG_HOTPLUG_CPU*/ /* Requires cpu_add_remove_lock to be held */ -static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen) +static int _cpu_up(unsigned int cpu, int tasks_frozen) { int ret, nr_calls = 0; void *hcpu = (void *)(long)cpu; @@ -438,7 +438,7 @@ static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen) return ret; } -int __cpuinit cpu_up(unsigned int cpu) +int cpu_up(unsigned int cpu) { int err = 0; @@ -636,7 +636,7 @@ core_initcall(cpu_hotplug_pm_sync_init); * It must be called by the arch code on the new cpu, before the new cpu * enables interrupts and before the "boot" cpu returns from __cpu_up(). */ -void __cpuinit notify_cpu_starting(unsigned int cpu) +void notify_cpu_starting(unsigned int cpu) { unsigned long val = CPU_STARTING; diff --git a/kernel/events/core.c b/kernel/events/core.c index 9dbed267efb5a..bb0fb2f862f5e 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7867,7 +7867,7 @@ static void __init perf_event_init_all_cpus(void) } } -static void __cpuinit perf_event_init_cpu(int cpu) +static void perf_event_init_cpu(int cpu) { struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); @@ -7999,7 +7999,7 @@ static struct notifier_block perf_reboot_notifier = { .priority = INT_MIN, }; -static int __cpuinit +static int perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (long)hcpu; diff --git a/kernel/fork.c b/kernel/fork.c index a743abbfb69da..aece712421540 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1615,7 +1615,7 @@ static inline void init_idle_pids(struct pid_link *links) } } -struct task_struct * __cpuinit fork_idle(int cpu) +struct task_struct * fork_idle(int cpu) { struct task_struct *task; task = copy_process(CLONE_VM, 0, 0, NULL, &init_struct_pid, 0); diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index a5f4dfeeab86b..17b82e5812fb7 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -1685,7 +1685,7 @@ SYSCALL_DEFINE2(nanosleep, struct timespec __user *, rqtp, /* * Functions related to boot-time initialization: */ -static void __cpuinit init_hrtimers_cpu(int cpu) +static void init_hrtimers_cpu(int cpu) { struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); int i; @@ -1766,7 +1766,7 @@ static void migrate_hrtimers(int scpu) #endif /* CONFIG_HOTPLUG_CPU */ -static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self, +static int hrtimer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { int scpu = (long)hcpu; @@ -1799,7 +1799,7 @@ static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata hrtimers_nb = { +static struct notifier_block hrtimers_nb = { .notifier_call = hrtimer_cpu_notify, }; diff --git a/kernel/printk.c b/kernel/printk.c index d737ae0bc8a0d..99823f8057c33 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -2310,13 +2310,13 @@ void resume_console(void) console_unlock(); } -static void __cpuinit console_flush(struct work_struct *work) +static void console_flush(struct work_struct *work) { console_lock(); console_unlock(); } -static __cpuinitdata DECLARE_WORK(console_cpu_notify_work, console_flush); +static DECLARE_WORK(console_cpu_notify_work, console_flush); /** * console_cpu_notify - print deferred console messages after CPU hotplug @@ -2332,7 +2332,7 @@ static __cpuinitdata DECLARE_WORK(console_cpu_notify_work, console_flush); * Special handling must be done for cases invoked from an atomic context, * as we can't be taking the console semaphore here. */ -static int __cpuinit console_cpu_notify(struct notifier_block *self, +static int console_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { switch (action) { diff --git a/kernel/profile.c b/kernel/profile.c index 594dff58c6078..4c0cb955f33c9 100644 --- a/kernel/profile.c +++ b/kernel/profile.c @@ -331,7 +331,7 @@ static void do_profile_hits(int type, void *__pc, unsigned int nr_hits) put_cpu(); } -static int __cpuinit profile_cpu_callback(struct notifier_block *info, +static int profile_cpu_callback(struct notifier_block *info, unsigned long action, void *__cpu) { int node, cpu = (unsigned long)__cpu; diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c index e1f3a8c967243..ceaa5d55671fc 100644 --- a/kernel/rcutorture.c +++ b/kernel/rcutorture.c @@ -1514,7 +1514,7 @@ rcu_torture_shutdown(void *arg) * Execute random CPU-hotplug operations at the interval specified * by the onoff_interval. */ -static int __cpuinit +static int rcu_torture_onoff(void *arg) { int cpu; @@ -1596,7 +1596,7 @@ rcu_torture_onoff(void *arg) return 0; } -static int __cpuinit +static int rcu_torture_onoff_init(void) { int ret; @@ -1639,7 +1639,7 @@ static void rcu_torture_onoff_cleanup(void) * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then * induces a CPU stall for the time specified by stall_cpu. */ -static int __cpuinit rcu_torture_stall(void *args) +static int rcu_torture_stall(void *args) { unsigned long stop_at; diff --git a/kernel/rcutree.c b/kernel/rcutree.c index c5cc4f1bce80f..20a56ced31015 100644 --- a/kernel/rcutree.c +++ b/kernel/rcutree.c @@ -2996,7 +2996,7 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp) * can accept some slop in the rsp->completed access due to the fact * that this CPU cannot possibly have any RCU callbacks in flight yet. */ -static void __cpuinit +static void rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible) { unsigned long flags; @@ -3049,7 +3049,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible) mutex_unlock(&rsp->onoff_mutex); } -static void __cpuinit rcu_prepare_cpu(int cpu) +static void rcu_prepare_cpu(int cpu) { struct rcu_state *rsp; @@ -3061,7 +3061,7 @@ static void __cpuinit rcu_prepare_cpu(int cpu) /* * Handle CPU online/offline notification events. */ -static int __cpuinit rcu_cpu_notify(struct notifier_block *self, +static int rcu_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { long cpu = (long)hcpu; diff --git a/kernel/rcutree.h b/kernel/rcutree.h index b07ec0473883d..f362084a9c90b 100644 --- a/kernel/rcutree.h +++ b/kernel/rcutree.h @@ -523,10 +523,10 @@ static void invoke_rcu_callbacks_kthread(void); static bool rcu_is_callbacks_kthread(void); #ifdef CONFIG_RCU_BOOST static void rcu_preempt_do_callbacks(void); -static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp, +static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp, struct rcu_node *rnp); #endif /* #ifdef CONFIG_RCU_BOOST */ -static void __cpuinit rcu_prepare_kthreads(int cpu); +static void rcu_prepare_kthreads(int cpu); static void rcu_prepare_for_idle_init(int cpu); static void rcu_cleanup_after_idle(int cpu); static void rcu_prepare_for_idle(int cpu); diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h index 104bb6b314f78..eb0adea7e0147 100644 --- a/kernel/rcutree_plugin.h +++ b/kernel/rcutree_plugin.h @@ -1327,7 +1327,7 @@ static void rcu_preempt_boost_start_gp(struct rcu_node *rnp) * already exist. We only create this kthread for preemptible RCU. * Returns zero if all is well, a negated errno otherwise. */ -static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp, +static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp, struct rcu_node *rnp) { int rnp_index = rnp - &rsp->node[0]; @@ -1482,7 +1482,7 @@ static int __init rcu_spawn_kthreads(void) } early_initcall(rcu_spawn_kthreads); -static void __cpuinit rcu_prepare_kthreads(int cpu) +static void rcu_prepare_kthreads(int cpu) { struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, cpu); struct rcu_node *rnp = rdp->mynode; @@ -1524,7 +1524,7 @@ static int __init rcu_scheduler_really_started(void) } early_initcall(rcu_scheduler_really_started); -static void __cpuinit rcu_prepare_kthreads(int cpu) +static void rcu_prepare_kthreads(int cpu) { } diff --git a/kernel/relay.c b/kernel/relay.c index b91488ba2e5a7..5001c9887db1a 100644 --- a/kernel/relay.c +++ b/kernel/relay.c @@ -516,7 +516,7 @@ static void setup_callbacks(struct rchan *chan, * * Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD) */ -static int __cpuinit relay_hotcpu_callback(struct notifier_block *nb, +static int relay_hotcpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu) { diff --git a/kernel/sched/core.c b/kernel/sched/core.c index dd5d2281dd210..c8659d8c0a0a5 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5449,7 +5449,7 @@ static struct notifier_block migration_notifier = { .priority = CPU_PRI_MIGRATION, }; -static void __cpuinit set_cpu_rq_start_time(void) +static void set_cpu_rq_start_time(void) { int cpu = smp_processor_id(); struct rq *rq = cpu_rq(cpu); diff --git a/kernel/smp.c b/kernel/smp.c index a835ecba883d2..7a2c5e3ebe9f8 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -78,7 +78,7 @@ hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu) return NOTIFY_OK; } -static struct notifier_block __cpuinitdata hotplug_cfd_notifier = { +static struct notifier_block hotplug_cfd_notifier = { .notifier_call = hotplug_cfd, }; diff --git a/kernel/smpboot.c b/kernel/smpboot.c index 7020eecb398bd..8d590db6d9e2a 100644 --- a/kernel/smpboot.c +++ b/kernel/smpboot.c @@ -24,7 +24,7 @@ */ static DEFINE_PER_CPU(struct task_struct *, idle_threads); -struct task_struct * __cpuinit idle_thread_get(unsigned int cpu) +struct task_struct * idle_thread_get(unsigned int cpu) { struct task_struct *tsk = per_cpu(idle_threads, cpu); diff --git a/kernel/softirq.c b/kernel/softirq.c index 96a4c2828ce65..9de0f2730c080 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -712,7 +712,7 @@ static void takeover_tasklets(unsigned int cpu) } #endif /* CONFIG_HOTPLUG_CPU */ -static int __cpuinit cpu_callback(struct notifier_block *nfb, +static int cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { @@ -727,7 +727,7 @@ static int __cpuinit cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata cpu_nfb = { +static struct notifier_block cpu_nfb = { .notifier_call = cpu_callback }; diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 228eba5ccc72e..c0e1b0e56d3e5 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -331,7 +331,7 @@ static int __init tick_nohz_full_setup(char *str) } __setup("nohz_full=", tick_nohz_full_setup); -static int __cpuinit tick_nohz_cpu_down_callback(struct notifier_block *nfb, +static int tick_nohz_cpu_down_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { diff --git a/kernel/timer.c b/kernel/timer.c index bbc99703765fb..0524f8104d418 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -1572,11 +1572,11 @@ signed long __sched schedule_timeout_uninterruptible(signed long timeout) } EXPORT_SYMBOL(schedule_timeout_uninterruptible); -static int __cpuinit init_timers_cpu(int cpu) +static int init_timers_cpu(int cpu) { int j; struct tvec_base *base; - static char __cpuinitdata tvec_base_done[NR_CPUS + 1]; + static char tvec_base_done[NR_CPUS + 1]; if (!tvec_base_done[cpu]) { static char boot_done; @@ -1659,7 +1659,7 @@ static void migrate_timer_list(struct tvec_base *new_base, struct list_head *hea } } -static void __cpuinit migrate_timers(int cpu) +static void migrate_timers(int cpu) { struct tvec_base *old_base; struct tvec_base *new_base; @@ -1692,7 +1692,7 @@ static void __cpuinit migrate_timers(int cpu) } #endif /* CONFIG_HOTPLUG_CPU */ -static int __cpuinit timer_cpu_notify(struct notifier_block *self, +static int timer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { long cpu = (long)hcpu; @@ -1717,7 +1717,7 @@ static int __cpuinit timer_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata timers_nb = { +static struct notifier_block timers_nb = { .notifier_call = timer_cpu_notify, }; diff --git a/kernel/workqueue.c b/kernel/workqueue.c index c67dc9c89f3b2..21d7ce5b3c0b0 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -4755,7 +4755,7 @@ static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu) * Workqueues should be brought up before normal priority CPU notifiers. * This will be registered high priority CPU notifier. */ -static int __cpuinit workqueue_cpu_up_callback(struct notifier_block *nfb, +static int workqueue_cpu_up_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { @@ -4808,7 +4808,7 @@ static int __cpuinit workqueue_cpu_up_callback(struct notifier_block *nfb, * Workqueues should be brought down after normal priority CPU notifiers. * This will be registered as low priority CPU notifier. */ -static int __cpuinit workqueue_cpu_down_callback(struct notifier_block *nfb, +static int workqueue_cpu_down_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { diff --git a/lib/earlycpio.c b/lib/earlycpio.c index 8078ef49cb793..7aa7ce250c94d 100644 --- a/lib/earlycpio.c +++ b/lib/earlycpio.c @@ -63,7 +63,7 @@ enum cpio_fields { * the match returned an empty filename string. */ -struct cpio_data __cpuinit find_cpio_data(const char *path, void *data, +struct cpio_data find_cpio_data(const char *path, void *data, size_t len, long *offset) { const size_t cpio_header_len = 8*C_NFIELDS - 2; diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c index ba6085d9c7411..867f2da6d697d 100644 --- a/lib/percpu_counter.c +++ b/lib/percpu_counter.c @@ -158,7 +158,7 @@ static void compute_batch_value(void) percpu_counter_batch = max(32, nr*2); } -static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb, +static int percpu_counter_hotcpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu) { #ifdef CONFIG_HOTPLUG_CPU diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 904244a74e43b..356caf075c918 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2565,7 +2565,7 @@ static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu) spin_unlock(&memcg->pcp_counter_lock); } -static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb, +static int memcg_cpu_hotplug_callback(struct notifier_block *nb, unsigned long action, void *hcpu) { diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 7474cf43110cd..e0ec4b74f5ecd 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -1761,7 +1761,7 @@ void writeback_set_ratelimit(void) ratelimit_pages = 16; } -static int __cpuinit +static int ratelimit_handler(struct notifier_block *self, unsigned long action, void *hcpu) { @@ -1776,7 +1776,7 @@ ratelimit_handler(struct notifier_block *self, unsigned long action, } } -static struct notifier_block __cpuinitdata ratelimit_nb = { +static struct notifier_block ratelimit_nb = { .notifier_call = ratelimit_handler, .next = NULL, }; diff --git a/mm/slab.c b/mm/slab.c index bd88411595b9b..a1b22c71c666c 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -787,7 +787,7 @@ static void next_reap_node(void) * the CPUs getting into lockstep and contending for the global cache chain * lock. */ -static void __cpuinit start_cpu_timer(int cpu) +static void start_cpu_timer(int cpu) { struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu); @@ -1180,7 +1180,7 @@ static int init_cache_node_node(int node) return 0; } -static void __cpuinit cpuup_canceled(long cpu) +static void cpuup_canceled(long cpu) { struct kmem_cache *cachep; struct kmem_cache_node *n = NULL; @@ -1245,7 +1245,7 @@ static void __cpuinit cpuup_canceled(long cpu) } } -static int __cpuinit cpuup_prepare(long cpu) +static int cpuup_prepare(long cpu) { struct kmem_cache *cachep; struct kmem_cache_node *n = NULL; @@ -1328,7 +1328,7 @@ static int __cpuinit cpuup_prepare(long cpu) return -ENOMEM; } -static int __cpuinit cpuup_callback(struct notifier_block *nfb, +static int cpuup_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { long cpu = (long)hcpu; @@ -1384,7 +1384,7 @@ static int __cpuinit cpuup_callback(struct notifier_block *nfb, return notifier_from_errno(err); } -static struct notifier_block __cpuinitdata cpucache_notifier = { +static struct notifier_block cpucache_notifier = { &cpuup_callback, NULL, 0 }; diff --git a/mm/slub.c b/mm/slub.c index 411d812471437..6d3b32f081b4d 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -3769,7 +3769,7 @@ int __kmem_cache_create(struct kmem_cache *s, unsigned long flags) * Use the cpu notifier to insure that the cpu slabs are flushed when * necessary. */ -static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb, +static int slab_cpuup_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { long cpu = (long)hcpu; @@ -3795,7 +3795,7 @@ static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata slab_notifier = { +static struct notifier_block slab_notifier = { .notifier_call = slab_cpuup_callback }; diff --git a/mm/vmstat.c b/mm/vmstat.c index c4a665630d4de..8f3e015b42759 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1281,7 +1281,7 @@ static void vmstat_update(struct work_struct *w) round_jiffies_relative(sysctl_stat_interval)); } -static void __cpuinit start_cpu_timer(int cpu) +static void start_cpu_timer(int cpu) { struct delayed_work *work = &per_cpu(vmstat_work, cpu); @@ -1293,7 +1293,7 @@ static void __cpuinit start_cpu_timer(int cpu) * Use the cpu notifier to insure that the thresholds are recalculated * when necessary. */ -static int __cpuinit vmstat_cpuup_callback(struct notifier_block *nfb, +static int vmstat_cpuup_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { @@ -1325,7 +1325,7 @@ static int __cpuinit vmstat_cpuup_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata vmstat_notifier = +static struct notifier_block vmstat_notifier = { &vmstat_cpuup_callback, NULL, 0 }; #endif diff --git a/net/core/flow.c b/net/core/flow.c index f74230f808675..9a2151f5d5935 100644 --- a/net/core/flow.c +++ b/net/core/flow.c @@ -403,7 +403,7 @@ void flow_cache_flush_deferred(void) schedule_work(&flow_cache_flush_work); } -static int __cpuinit flow_cache_cpu_prepare(struct flow_cache *fc, int cpu) +static int flow_cache_cpu_prepare(struct flow_cache *fc, int cpu) { struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu); size_t sz = sizeof(struct hlist_head) * flow_cache_hash_size(fc); @@ -421,7 +421,7 @@ static int __cpuinit flow_cache_cpu_prepare(struct flow_cache *fc, int cpu) return 0; } -static int __cpuinit flow_cache_cpu(struct notifier_block *nfb, +static int flow_cache_cpu(struct notifier_block *nfb, unsigned long action, void *hcpu) { diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c index 4fe76ff214c2a..cd5b8ec9be045 100644 --- a/net/iucv/iucv.c +++ b/net/iucv/iucv.c @@ -621,7 +621,7 @@ static void iucv_disable(void) put_online_cpus(); } -static int __cpuinit iucv_cpu_notify(struct notifier_block *self, +static int iucv_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { cpumask_t cpumask; diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 2fa73ee7942f3..dd907ee6e2ac9 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -868,23 +868,23 @@ static void check_section(const char *modname, struct elf_info *elf, #define ALL_INIT_DATA_SECTIONS \ ".init.setup$", ".init.rodata$", \ - ".cpuinit.rodata$", ".meminit.rodata$", \ - ".init.data$", ".cpuinit.data$", ".meminit.data$" + ".meminit.rodata$", \ + ".init.data$", ".meminit.data$" #define ALL_EXIT_DATA_SECTIONS \ - ".exit.data$", ".cpuexit.data$", ".memexit.data$" + ".exit.data$", ".memexit.data$" #define ALL_INIT_TEXT_SECTIONS \ - ".init.text$", ".cpuinit.text$", ".meminit.text$" + ".init.text$", ".meminit.text$" #define ALL_EXIT_TEXT_SECTIONS \ - ".exit.text$", ".cpuexit.text$", ".memexit.text$" + ".exit.text$", ".memexit.text$" #define ALL_PCI_INIT_SECTIONS \ ".pci_fixup_early$", ".pci_fixup_header$", ".pci_fixup_final$", \ ".pci_fixup_enable$", ".pci_fixup_resume$", \ ".pci_fixup_resume_early$", ".pci_fixup_suspend$" -#define ALL_XXXINIT_SECTIONS CPU_INIT_SECTIONS, MEM_INIT_SECTIONS -#define ALL_XXXEXIT_SECTIONS CPU_EXIT_SECTIONS, MEM_EXIT_SECTIONS +#define ALL_XXXINIT_SECTIONS MEM_INIT_SECTIONS +#define ALL_XXXEXIT_SECTIONS MEM_EXIT_SECTIONS #define ALL_INIT_SECTIONS INIT_SECTIONS, ALL_XXXINIT_SECTIONS #define ALL_EXIT_SECTIONS EXIT_SECTIONS, ALL_XXXEXIT_SECTIONS @@ -893,11 +893,9 @@ static void check_section(const char *modname, struct elf_info *elf, #define TEXT_SECTIONS ".text$" #define INIT_SECTIONS ".init.*" -#define CPU_INIT_SECTIONS ".cpuinit.*" #define MEM_INIT_SECTIONS ".meminit.*" #define EXIT_SECTIONS ".exit.*" -#define CPU_EXIT_SECTIONS ".cpuexit.*" #define MEM_EXIT_SECTIONS ".memexit.*" /* init data sections */ @@ -985,48 +983,20 @@ const struct sectioncheck sectioncheck[] = { .mismatch = DATA_TO_ANY_EXIT, .symbol_white_list = { DEFAULT_SYMBOL_WHITE_LIST, NULL }, }, -/* Do not reference init code/data from cpuinit/meminit code/data */ +/* Do not reference init code/data from meminit code/data */ { .fromsec = { ALL_XXXINIT_SECTIONS, NULL }, .tosec = { INIT_SECTIONS, NULL }, .mismatch = XXXINIT_TO_SOME_INIT, .symbol_white_list = { DEFAULT_SYMBOL_WHITE_LIST, NULL }, }, -/* Do not reference cpuinit code/data from meminit code/data */ -{ - .fromsec = { MEM_INIT_SECTIONS, NULL }, - .tosec = { CPU_INIT_SECTIONS, NULL }, - .mismatch = XXXINIT_TO_SOME_INIT, - .symbol_white_list = { DEFAULT_SYMBOL_WHITE_LIST, NULL }, -}, -/* Do not reference meminit code/data from cpuinit code/data */ -{ - .fromsec = { CPU_INIT_SECTIONS, NULL }, - .tosec = { MEM_INIT_SECTIONS, NULL }, - .mismatch = XXXINIT_TO_SOME_INIT, - .symbol_white_list = { DEFAULT_SYMBOL_WHITE_LIST, NULL }, -}, -/* Do not reference exit code/data from cpuexit/memexit code/data */ +/* Do not reference exit code/data from memexit code/data */ { .fromsec = { ALL_XXXEXIT_SECTIONS, NULL }, .tosec = { EXIT_SECTIONS, NULL }, .mismatch = XXXEXIT_TO_SOME_EXIT, .symbol_white_list = { DEFAULT_SYMBOL_WHITE_LIST, NULL }, }, -/* Do not reference cpuexit code/data from memexit code/data */ -{ - .fromsec = { MEM_EXIT_SECTIONS, NULL }, - .tosec = { CPU_EXIT_SECTIONS, NULL }, - .mismatch = XXXEXIT_TO_SOME_EXIT, - .symbol_white_list = { DEFAULT_SYMBOL_WHITE_LIST, NULL }, -}, -/* Do not reference memexit code/data from cpuexit code/data */ -{ - .fromsec = { CPU_EXIT_SECTIONS, NULL }, - .tosec = { MEM_EXIT_SECTIONS, NULL }, - .mismatch = XXXEXIT_TO_SOME_EXIT, - .symbol_white_list = { DEFAULT_SYMBOL_WHITE_LIST, NULL }, -}, /* Do not use exit code/data from init code */ { .fromsec = { ALL_INIT_SECTIONS, NULL }, @@ -1095,7 +1065,7 @@ static const struct sectioncheck *section_mismatch( * Pattern 2: * Many drivers utilise a *driver container with references to * add, remove, probe functions etc. - * These functions may often be marked __cpuinit and we do not want to + * These functions may often be marked and we do not want to * warn here. * the pattern is identified by: * tosec = init or exit section @@ -1255,7 +1225,6 @@ static Elf_Sym *find_elf_symbol2(struct elf_info *elf, Elf_Addr addr, /* * Convert a section name to the function/data attribute * .init.text => __init - * .cpuinit.data => __cpudata * .memexitconst => __memconst * etc. * From f978a5610bfca47772de731b036713a1acfe5e9a Mon Sep 17 00:00:00 2001 From: "www.anuprita804@gmail.com" Date: Sat, 19 Aug 2017 00:27:11 +0200 Subject: [PATCH 86/86] revert oneplus changes for better eas compatibility --- arch/arm64/configs/oneplus2_defconfig | 2 +- drivers/android/binder.c | 4 ---- drivers/cpufreq/cpu-boost.c | 5 ----- fs/open.c | 2 -- include/linux/sched/sysctl.h | 3 --- kernel/cgroup.c | 3 --- kernel/sched/sched.h | 3 --- mm/memory.c | 3 --- 8 files changed, 1 insertion(+), 24 deletions(-) diff --git a/arch/arm64/configs/oneplus2_defconfig b/arch/arm64/configs/oneplus2_defconfig index 531693a85a4de..b33447f962596 100644 --- a/arch/arm64/configs/oneplus2_defconfig +++ b/arch/arm64/configs/oneplus2_defconfig @@ -507,7 +507,7 @@ CONFIG_MSM_SYSTEM_HEALTH_MONITOR=y CONFIG_QCOM_EARLY_RANDOM=y CONFIG_MSM_CORE_CTL_HELPER=y CONFIG_MSM_PERFORMANCE=y -CONFIG_ONEPLUS_HCUBE=m +CONFIG_ONEPLUS_HCUBE=n CONFIG_QCOM_NPA_DUMP=y CONFIG_MSM_TZ_LOG=y CONFIG_PARAM_READ_WRITE=y diff --git a/drivers/android/binder.c b/drivers/android/binder.c index e2ef66adf5197..7796abcfb6ed0 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -2293,7 +2293,6 @@ static int binder_thread_read(struct binder_proc *proc, binder_unlock(__func__); #if defined(VENDOR_EDIT) && defined(CONFIG_CGROUP_SCHED) - skip_cfs_throttle(0); #endif trace_binder_wait_for_work(wait_for_proc_work, @@ -2322,7 +2321,6 @@ static int binder_thread_read(struct binder_proc *proc, } #if defined(VENDOR_EDIT) && defined(CONFIG_CGROUP_SCHED) - skip_cfs_throttle(1); #endif binder_lock(__func__); @@ -2857,7 +2855,6 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) goto err_unlocked; #if defined(VENDOR_EDIT) && defined(CONFIG_CGROUP_SCHED) - skip_cfs_throttle(1); #endif binder_lock(__func__); thread = binder_get_thread(proc); @@ -2914,7 +2911,6 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) thread->looper &= ~BINDER_LOOPER_STATE_NEED_RETURN; binder_unlock(__func__); #if defined(VENDOR_EDIT) && defined(CONFIG_CGROUP_SCHED) - skip_cfs_throttle(0); #endif wait_event_interruptible(binder_user_error_wait, binder_stop_on_user_error < 2); if (ret && ret != -ERESTARTSYS) diff --git a/drivers/cpufreq/cpu-boost.c b/drivers/cpufreq/cpu-boost.c index b5b3fbf5d1c09..545d484b17596 100644 --- a/drivers/cpufreq/cpu-boost.c +++ b/drivers/cpufreq/cpu-boost.c @@ -48,11 +48,6 @@ static struct work_struct input_boost_work; static unsigned int boost_ms; module_param(boost_ms, uint, 0644); -#ifdef VENDOR_EDIT -extern unsigned int sysctl_thermal_aware_scheduling; -module_param(sysctl_thermal_aware_scheduling, uint, 0644); -#endif - static unsigned int sync_threshold; module_param(sync_threshold, uint, 0644); diff --git a/fs/open.c b/fs/open.c index 3dd51d4af04b5..da01069ad4d43 100644 --- a/fs/open.c +++ b/fs/open.c @@ -940,7 +940,6 @@ long do_sys_open(int dfd, const char __user *filename, int flags, umode_t mode) struct filename *tmp = getname(filename); int fd = PTR_ERR(tmp); - skip_cfs_throttle(1); if (!IS_ERR(tmp)) { fd = get_unused_fd_flags(flags); if (fd >= 0) { @@ -955,7 +954,6 @@ long do_sys_open(int dfd, const char __user *filename, int flags, umode_t mode) } putname(tmp); } - skip_cfs_throttle(0); return fd; } diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index b3e9aab6f5b46..aad9ce9ce47f0 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -49,9 +49,6 @@ extern unsigned int sysctl_sched_freq_account_wait_time; extern unsigned int sysctl_sched_migration_fixup; extern unsigned int sysctl_sched_heavy_task_pct; extern unsigned int sysctl_sched_enable_power_aware; -#ifdef VENDOR_EDIT -extern unsigned int sysctl_thermal_aware_scheduling; -#endif #if defined(CONFIG_SCHED_FREQ_INPUT) || defined(CONFIG_SCHED_HMP) extern unsigned int sysctl_sched_init_task_load_pct; diff --git a/kernel/cgroup.c b/kernel/cgroup.c index 1f59aaa60497a..f381a941492a2 100755 --- a/kernel/cgroup.c +++ b/kernel/cgroup.c @@ -2155,9 +2155,7 @@ static int attach_task_by_pid(struct cgroup *cgrp, u64 pid, bool threadgroup) const struct cred *cred = current_cred(), *tcred; int ret; - skip_cfs_throttle(1); if (!cgroup_lock_live_group(cgrp)) { - skip_cfs_throttle(0); return -ENODEV; } @@ -2235,7 +2233,6 @@ static int attach_task_by_pid(struct cgroup *cgrp, u64 pid, bool threadgroup) put_task_struct(tsk); out_unlock_cgroup: mutex_unlock(&cgroup_mutex); - skip_cfs_throttle(0); return ret; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 488a56e434b97..b5ef65d3e4c4e 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -192,9 +192,6 @@ struct cfs_rq; struct rt_rq; extern struct list_head task_groups; -#ifdef VENDOR_EDIT -void skip_cfs_throttle(int skip); -#endif struct cfs_bandwidth { #ifdef CONFIG_CFS_BANDWIDTH diff --git a/mm/memory.c b/mm/memory.c index a3646c281a0f5..678bd985cfdd1 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4196,16 +4196,13 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, struct mm_struct *mm; int ret; - skip_cfs_throttle(1); mm = get_task_mm(tsk); if (!mm) { - skip_cfs_throttle(0); return 0; } ret = __access_remote_vm(tsk, mm, addr, buf, len, write); mmput(mm); - skip_cfs_throttle(0); return ret; }