diff --git a/Documentation/devicetree/bindings/scheduler/sched-energy-costs.txt b/Documentation/devicetree/bindings/scheduler/sched-energy-costs.txt new file mode 100644 index 0000000000000..11216f09e596b --- /dev/null +++ b/Documentation/devicetree/bindings/scheduler/sched-energy-costs.txt @@ -0,0 +1,360 @@ +=========================================================== +Energy cost bindings for Energy Aware Scheduling +=========================================================== + +=========================================================== +1 - Introduction +=========================================================== + +This note specifies bindings required for energy-aware scheduling +(EAS)[1]. Historically, the scheduler's primary objective has been +performance. EAS aims to provide an alternative objective - energy +efficiency. EAS relies on a simple platform energy cost model to +guide scheduling decisions. The model only considers the CPU +subsystem. + +This note is aligned with the definition of the layout of physical +CPUs in the system as described in the ARM topology binding +description [2]. The concept is applicable to any system so long as +the cost model data is provided for those processing elements in +that system's topology that EAS is required to service. + +Processing elements refer to hardware threads, CPUs and clusters of +related CPUs in increasing order of hierarchy. + +EAS requires two key cost metrics - busy costs and idle costs. Busy +costs comprise of a list of compute capacities for the processing +element in question and the corresponding power consumption at that +capacity. Idle costs comprise of a list of power consumption values +for each idle state [C-state] that the processing element supports. +For a detailed description of these metrics, their derivation and +their use see [3]. + +These cost metrics are required for processing elements in all +scheduling domain levels that EAS is required to service. + +=========================================================== +2 - energy-costs node +=========================================================== + +Energy costs for the processing elements in scheduling domains that +EAS is required to service are defined in the energy-costs node +which acts as a container for the actual per processing element cost +nodes. A single energy-costs node is required for a given system. + +- energy-costs node + + Usage: Required + + Description: The energy-costs node is a container node and + it's sub-nodes describe costs for each processing element at + all scheduling domain levels that EAS is required to + service. + + Node name must be "energy-costs". + + The energy-costs node's parent node must be the cpus node. + + The energy-costs node's child nodes can be: + + - one or more cost nodes. + + Any other configuration is considered invalid. + +The energy-costs node can only contain a single type of child node +whose bindings are described in paragraph 4. + +=========================================================== +3 - energy-costs node child nodes naming convention +=========================================================== + +energy-costs child nodes must follow a naming convention where the +node name must be "thread-costN", "core-costN", "cluster-costN" +depending on whether the costs in the node are for a thread, core or +cluster. N (where N = {0, 1, ...}) is the node number and has no +bearing to the OS' logical thread, core or cluster index. + +=========================================================== +4 - cost node bindings +=========================================================== + +Bindings for cost nodes are defined as follows: + +- cluster-cost node + + Description: must be declared within an energy-costs node. A + system can contain multiple clusters and each cluster + serviced by EAS must have a corresponding cluster-costs + node. + + The cluster-cost node name must be "cluster-costN" as + described in 3 above. + + A cluster-cost node must be a leaf node with no children. + + Properties for cluster-cost nodes are described in paragraph + 5 below. + + Any other configuration is considered invalid. + +- core-cost node + + Description: must be declared within an energy-costs node. A + system can contain multiple cores and each core serviced by + EAS must have a corresponding core-cost node. + + The core-cost node name must be "core-costN" as described in + 3 above. + + A core-cost node must be a leaf node with no children. + + Properties for core-cost nodes are described in paragraph + 5 below. + + Any other configuration is considered invalid. + +- thread-cost node + + Description: must be declared within an energy-costs node. A + system can contain cores with multiple hardware threads and + each thread serviced by EAS must have a corresponding + thread-cost node. + + The core-cost node name must be "core-costN" as described in + 3 above. + + A core-cost node must be a leaf node with no children. + + Properties for thread-cost nodes are described in paragraph + 5 below. + + Any other configuration is considered invalid. + +=========================================================== +5 - Cost node properties +========================================================== + +All cost node types must have only the following properties: + +- busy-cost-data + + Usage: required + Value type: An array of 2-item tuples. Each item is of type + u32. + Definition: The first item in the tuple is the capacity + value as described in [3]. The second item in the tuple is + the energy cost value as described in [3]. + +- idle-cost-data + + Usage: required + Value type: An array of 1-item tuples. The item is of type + u32. + Definition: The item in the tuple is the energy cost value + as described in [3]. + +=========================================================== +4 - Extensions to the cpu node +=========================================================== + +The cpu node is extended with a property that establishes the +connection between the processing element represented by the cpu +node and the cost-nodes associated with this processing element. + +The connection is expressed in line with the topological hierarchy +that this processing element belongs to starting with the level in +the hierarchy that this processing element itself belongs to through +to the highest level that EAS is required to service. The +connection cannot be sparse and must be contiguous from the +processing element's level through to the highest desired level. The +highest desired level must be the same for all processing elements. + +Example: Given that a cpu node may represent a thread that is a part +of a core, this property may contain multiple elements which +associate the thread with cost nodes describing the costs for the +thread itself, the core the thread belongs to, the cluster the core +belongs to and so on. The elements must be ordered from the lowest +level nodes to the highest desired level that EAS must service. The +highest desired level must be the same for all cpu nodes. The +elements must not be sparse: there must be elements for the current +thread, the next level of hierarchy (core) and so on without any +'holes'. + +Example: Given that a cpu node may represent a core that is a part +of a cluster of related cpus this property may contain multiple +elements which associate the core with cost nodes describing the +costs for the core itself, the cluster the core belongs to and so +on. The elements must be ordered from the lowest level nodes to the +highest desired level that EAS must service. The highest desired +level must be the same for all cpu nodes. The elements must not be +sparse: there must be elements for the current thread, the next +level of hierarchy (core) and so on without any 'holes'. + +If the system comprises of hierarchical clusters of clusters, this +property will contain multiple associations with the relevant number +of cluster elements in hierarchical order. + +Property added to the cpu node: + +- sched-energy-costs + + Usage: required + Value type: List of phandles + Definition: a list of phandles to specific cost nodes in the + energy-costs parent node that correspond to the processing + element represented by this cpu node in hierarchical order + of topology. + + The order of phandles in the list is significant. The first + phandle is to the current processing element's own cost + node. Subsequent phandles are to higher hierarchical level + cost nodes up until the maximum level that EAS is to + service. + + All cpu nodes must have the same highest level cost node. + + The phandle list must not be sparsely populated with handles + to non-contiguous hierarchical levels. See commentary above + for clarity. + + Any other configuration is invalid. + +=========================================================== +5 - Example dts +=========================================================== + +Example 1 (ARM 64-bit, 6-cpu system, two clusters of cpus, one +cluster of 2 Cortex-A57 cpus, one cluster of 4 Cortex-A53 cpus): + +cpus { + #address-cells = <2>; + #size-cells = <0>; + . + . + . + A57_0: cpu@0 { + compatible = "arm,cortex-a57","arm,armv8"; + reg = <0x0 0x0>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A57_L2>; + clocks = <&scpi_dvfs 0>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + sched-energy-costs = <&CPU_COST_0 &CLUSTER_COST_0>; + }; + + A57_1: cpu@1 { + compatible = "arm,cortex-a57","arm,armv8"; + reg = <0x0 0x1>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A57_L2>; + clocks = <&scpi_dvfs 0>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + sched-energy-costs = <&CPU_COST_0 &CLUSTER_COST_0>; + }; + + A53_0: cpu@100 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x100>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + clocks = <&scpi_dvfs 1>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; + }; + + A53_1: cpu@101 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x101>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + clocks = <&scpi_dvfs 1>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; + }; + + A53_2: cpu@102 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x102>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + clocks = <&scpi_dvfs 1>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; + }; + + A53_3: cpu@103 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x103>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + clocks = <&scpi_dvfs 1>; + cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; + }; + + energy-costs { + CPU_COST_0: core-cost0 { + busy-cost-data = < + 417 168 + 579 251 + 744 359 + 883 479 + 1024 616 + >; + idle-cost-data = < + 15 + 0 + >; + }; + CPU_COST_1: core-cost1 { + busy-cost-data = < + 235 33 + 302 46 + 368 61 + 406 76 + 447 93 + >; + idle-cost-data = < + 6 + 0 + >; + }; + CLUSTER_COST_0: cluster-cost0 { + busy-cost-data = < + 417 24 + 579 32 + 744 43 + 883 49 + 1024 64 + >; + idle-cost-data = < + 65 + 24 + >; + }; + CLUSTER_COST_1: cluster-cost1 { + busy-cost-data = < + 235 26 + 303 30 + 368 39 + 406 47 + 447 57 + >; + idle-cost-data = < + 56 + 17 + >; + }; + }; +}; + +=============================================================================== +[1] https://lkml.org/lkml/2015/5/12/728 +[2] Documentation/devicetree/bindings/topology.txt +[3] Documentation/scheduler/sched-energy.txt diff --git a/Documentation/scheduler/sched-energy.txt b/Documentation/scheduler/sched-energy.txt new file mode 100644 index 0000000000000..dab2f9088b336 --- /dev/null +++ b/Documentation/scheduler/sched-energy.txt @@ -0,0 +1,362 @@ +Energy cost model for energy-aware scheduling (EXPERIMENTAL) + +Introduction +============= + +The basic energy model uses platform energy data stored in sched_group_energy +data structures attached to the sched_groups in the sched_domain hierarchy. The +energy cost model offers two functions that can be used to guide scheduling +decisions: + +1. static unsigned int sched_group_energy(struct energy_env *eenv) +2. static int energy_diff(struct energy_env *eenv) + +sched_group_energy() estimates the energy consumed by all cpus in a specific +sched_group including any shared resources owned exclusively by this group of +cpus. Resources shared with other cpus are excluded (e.g. later level caches). + +energy_diff() estimates the total energy impact of a utilization change. That +is, adding, removing, or migrating utilization (tasks). + +Both functions use a struct energy_env to specify the scenario to be evaluated: + + struct energy_env { + struct sched_group *sg_top; + struct sched_group *sg_cap; + int cap_idx; + int util_delta; + int src_cpu; + int dst_cpu; + int energy; + }; + +sg_top: sched_group to be evaluated. Not used by energy_diff(). + +sg_cap: sched_group covering the cpus in the same frequency domain. Set by +sched_group_energy(). + +cap_idx: Capacity state to be used for energy calculations. Set by +find_new_capacity(). + +util_delta: Amount of utilization to be added, removed, or migrated. + +src_cpu: Source cpu from where 'util_delta' utilization is removed. Should be +-1 if no source (e.g. task wake-up). + +dst_cpu: Destination cpu where 'util_delta' utilization is added. Should be -1 +if utilization is removed (e.g. terminating tasks). + +energy: Result of sched_group_energy(). + +The metric used to represent utilization is the actual per-entity running time +averaged over time using a geometric series. Very similar to the existing +per-entity load-tracking, but _not_ scaled by task priority and capped by the +capacity of the cpu. The latter property does mean that utilization may +underestimate the compute requirements for task on fully/over utilized cpus. +The greatest potential for energy savings without affecting performance too much +is scenarios where the system isn't fully utilized. If the system is deemed +fully utilized load-balancing should be done with task load (includes task +priority) instead in the interest of fairness and performance. + + +Background and Terminology +=========================== + +To make it clear from the start: + +energy = [joule] (resource like a battery on powered devices) +power = energy/time = [joule/second] = [watt] + +The goal of energy-aware scheduling is to minimize energy, while still getting +the job done. That is, we want to maximize: + + performance [inst/s] + -------------------- + power [W] + +which is equivalent to minimizing: + + energy [J] + ----------- + instruction + +while still getting 'good' performance. It is essentially an alternative +optimization objective to the current performance-only objective for the +scheduler. This alternative considers two objectives: energy-efficiency and +performance. Hence, there needs to be a user controllable knob to switch the +objective. Since it is early days, this is currently a sched_feature +(ENERGY_AWARE). + +The idea behind introducing an energy cost model is to allow the scheduler to +evaluate the implications of its decisions rather than applying energy-saving +techniques blindly that may only have positive effects on some platforms. At +the same time, the energy cost model must be as simple as possible to minimize +the scheduler latency impact. + +Platform topology +------------------ + +The system topology (cpus, caches, and NUMA information, not peripherals) is +represented in the scheduler by the sched_domain hierarchy which has +sched_groups attached at each level that covers one or more cpus (see +sched-domains.txt for more details). To add energy awareness to the scheduler +we need to consider power and frequency domains. + +Power domain: + +A power domain is a part of the system that can be powered on/off +independently. Power domains are typically organized in a hierarchy where you +may be able to power down just a cpu or a group of cpus along with any +associated resources (e.g. shared caches). Powering up a cpu means that all +power domains it is a part of in the hierarchy must be powered up. Hence, it is +more expensive to power up the first cpu that belongs to a higher level power +domain than powering up additional cpus in the same high level domain. Two +level power domain hierarchy example: + + Power source + +-------------------------------+----... +per group PD G G + | +----------+ | + +--------+-------| Shared | (other groups) +per-cpu PD G G | resource | + | | +----------+ + +-------+ +-------+ + | CPU 0 | | CPU 1 | + +-------+ +-------+ + +Frequency domain: + +Frequency domains (P-states) typically cover the same group of cpus as one of +the power domain levels. That is, there might be several smaller power domains +sharing the same frequency (P-state) or there might be a power domain spanning +multiple frequency domains. + +From a scheduling point of view there is no need to know the actual frequencies +[Hz]. All the scheduler cares about is the compute capacity available at the +current state (P-state) the cpu is in and any other available states. For that +reason, and to also factor in any cpu micro-architecture differences, compute +capacity scaling states are called 'capacity states' in this document. For SMP +systems this is equivalent to P-states. For mixed micro-architecture systems +(like ARM big.LITTLE) it is P-states scaled according to the micro-architecture +performance relative to the other cpus in the system. + +Energy modelling: +------------------ + +Due to the hierarchical nature of the power domains, the most obvious way to +model energy costs is therefore to associate power and energy costs with +domains (groups of cpus). Energy costs of shared resources are associated with +the group of cpus that share the resources, only the cost of powering the +cpu itself and any private resources (e.g. private L1 caches) is associated +with the per-cpu groups (lowest level). + +For example, for an SMP system with per-cpu power domains and a cluster level +(group of cpus) power domain we get the overall energy costs to be: + + energy = energy_cluster + n * energy_cpu + +where 'n' is the number of cpus powered up and energy_cluster is the cost paid +as soon as any cpu in the cluster is powered up. + +The power and frequency domains can naturally be mapped onto the existing +sched_domain hierarchy and sched_groups by adding the necessary data to the +existing data structures. + +The energy model considers energy consumption from two contributors (shown in +the illustration below): + +1. Busy energy: Energy consumed while a cpu and the higher level groups that it +belongs to are busy running tasks. Busy energy is associated with the state of +the cpu, not an event. The time the cpu spends in this state varies. Thus, the +most obvious platform parameter for this contribution is busy power +(energy/time). + +2. Idle energy: Energy consumed while a cpu and higher level groups that it +belongs to are idle (in a C-state). Like busy energy, idle energy is associated +with the state of the cpu. Thus, the platform parameter for this contribution +is idle power (energy/time). + +Energy consumed during transitions from an idle-state (C-state) to a busy state +(P-state) or going the other way is ignored by the model to simplify the energy +model calculations. + + + Power + ^ + | busy->idle idle->busy + | transition transition + | + | _ __ + | / \ / \__________________ + |______________/ \ / + | \ / + | Busy \ Idle / Busy + | low P-state \____________/ high P-state + | + +------------------------------------------------------------> time + +Busy |--------------| |-----------------| + +Wakeup |------| |------| + +Idle |------------| + + +The basic algorithm +==================== + +The basic idea is to determine the total energy impact when utilization is +added or removed by estimating the impact at each level in the sched_domain +hierarchy starting from the bottom (sched_group contains just a single cpu). +The energy cost comes from busy time (sched_group is awake because one or more +cpus are busy) and idle time (in an idle-state). Energy model numbers account +for energy costs associated with all cpus in the sched_group as a group. + + for_each_domain(cpu, sd) { + sg = sched_group_of(cpu) + energy_before = curr_util(sg) * busy_power(sg) + + (1-curr_util(sg)) * idle_power(sg) + energy_after = new_util(sg) * busy_power(sg) + + (1-new_util(sg)) * idle_power(sg) + energy_diff += energy_before - energy_after + + } + + return energy_diff + +{curr, new}_util: The cpu utilization at the lowest level and the overall +non-idle time for the entire group for higher levels. Utilization is in the +range 0.0 to 1.0 in the pseudo-code. + +busy_power: The power consumption of the sched_group. + +idle_power: The power consumption of the sched_group when idle. + +Note: It is a fundamental assumption that the utilization is (roughly) scale +invariant. Task utilization tracking factors in any frequency scaling and +performance scaling differences due to difference cpu microarchitectures such +that task utilization can be used across the entire system. + + +Platform energy data +===================== + +struct sched_group_energy can be attached to sched_groups in the sched_domain +hierarchy and has the following members: + +cap_states: + List of struct capacity_state representing the supported capacity states + (P-states). struct capacity_state has two members: cap and power, which + represents the compute capacity and the busy_power of the state. The + list must be ordered by capacity low->high. + +nr_cap_states: + Number of capacity states in cap_states list. + +idle_states: + List of struct idle_state containing idle_state power cost for each + idle-state supported by the system orderd by shallowest state first. + All states must be included at all level in the hierarchy, i.e. a + sched_group spanning just a single cpu must also include coupled + idle-states (cluster states). In addition to the cpuidle idle-states, + the list must also contain an entry for the idling using the arch + default idle (arch_idle_cpu()). Despite this state may not be a true + hardware idle-state it is considered the shallowest idle-state in the + energy model and must be the first entry. cpus may enter this state + (possibly 'active idling') if cpuidle decides not enter a cpuidle + idle-state. Default idle may not be used when cpuidle is enabled. + In this case, it should just be a copy of the first cpuidle idle-state. + +nr_idle_states: + Number of idle states in idle_states list. + +There are no unit requirements for the energy cost data. Data can be normalized +with any reference, however, the normalization must be consistent across all +energy cost data. That is, one bogo-joule/watt must be the same quantity for +data, but we don't care what it is. + +A recipe for platform characterization +======================================= + +Obtaining the actual model data for a particular platform requires some way of +measuring power/energy. There isn't a tool to help with this (yet). This +section provides a recipe for use as reference. It covers the steps used to +characterize the ARM TC2 development platform. This sort of measurements is +expected to be done anyway when tuning cpuidle and cpufreq for a given +platform. + +The energy model needs two types of data (struct sched_group_energy holds +these) for each sched_group where energy costs should be taken into account: + +1. Capacity state information + +A list containing the compute capacity and power consumption when fully +utilized attributed to the group as a whole for each available capacity state. +At the lowest level (group contains just a single cpu) this is the power of the +cpu alone without including power consumed by resources shared with other cpus. +It basically needs to fit the basic modelling approach described in "Background +and Terminology" section: + + energy_system = energy_shared + n * energy_cpu + +for a system containing 'n' busy cpus. Only 'energy_cpu' should be included at +the lowest level. 'energy_shared' is included at the next level which +represents the group of cpus among which the resources are shared. + +This model is, of course, a simplification of reality. Thus, power/energy +attributions might not always exactly represent how the hardware is designed. +Also, busy power is likely to depend on the workload. It is therefore +recommended to use a representative mix of workloads when characterizing the +capacity states. + +If the group has no capacity scaling support, the list will contain a single +state where power is the busy power attributed to the group. The capacity +should be set to a default value (1024). + +When frequency domains include multiple power domains, the group representing +the frequency domain and all child groups share capacity states. This must be +indicated by setting the SD_SHARE_CAP_STATES sched_domain flag. All groups at +all levels that share the capacity state must have the list of capacity states +with the power set to the contribution of the individual group. + +2. Idle power information + +Stored in the idle_states list. The power number is the group idle power +consumption in each idle state as well when the group is idle but has not +entered an idle-state ('active idle' as mentioned earlier). Due to the way the +energy model is defined, the idle power of the deepest group idle state can +alternatively be accounted for in the parent group busy power. In that case the +group idle state power values are offset such that the idle power of the +deepest state is zero. It is less intuitive, but it is easier to measure as +idle power consumed by the group and the busy/idle power of the parent group +cannot be distinguished without per group measurement points. + +Measuring capacity states and idle power: + +The capacity states' capacity and power can be estimated by running a benchmark +workload at each available capacity state. By restricting the benchmark to run +on subsets of cpus it is possible to extrapolate the power consumption of +shared resources. + +ARM TC2 has two clusters of two and three cpus respectively. Each cluster has a +shared L2 cache. TC2 has on-chip energy counters per cluster. Running a +benchmark workload on just one cpu in a cluster means that power is consumed in +the cluster (higher level group) and a single cpu (lowest level group). Adding +another benchmark task to another cpu increases the power consumption by the +amount consumed by the additional cpu. Hence, it is possible to extrapolate the +cluster busy power. + +For platforms that don't have energy counters or equivalent instrumentation +built-in, it may be possible to use an external DAQ to acquire similar data. + +If the benchmark includes some performance score (for example sysbench cpu +benchmark), this can be used to record the compute capacity. + +Measuring idle power requires insight into the idle state implementation on the +particular platform. Specifically, if the platform has coupled idle-states (or +package states). To measure non-coupled per-cpu idle-states it is necessary to +keep one cpu busy to keep any shared resources alive to isolate the idle power +of the cpu from idle/busy power of the shared resources. The cpu can be tricked +into different per-cpu idle states by disabling the other states. Based on +various combinations of measurements with specific cpus busy and disabling +idle-states it is possible to extrapolate the idle-state power. diff --git a/Documentation/scheduler/sched-tune.txt b/Documentation/scheduler/sched-tune.txt new file mode 100644 index 0000000000000..cb795e643eba1 --- /dev/null +++ b/Documentation/scheduler/sched-tune.txt @@ -0,0 +1,367 @@ + Central, scheduler-driven, power-performance control + (EXPERIMENTAL) + +Abstract +======== + +The topic of a single simple power-performance tunable, that is wholly +scheduler centric, and has well defined and predictable properties has come up +on several occasions in the past [1,2]. With techniques such as a scheduler +driven DVFS [3], we now have a good framework for implementing such a tunable. +This document describes the overall ideas behind its design and implementation. + + +Table of Contents +================= + +1. Motivation +2. Introduction +3. Signal Boosting Strategy +4. OPP selection using boosted CPU utilization +5. Per task group boosting +6. Question and Answers + - What about "auto" mode? + - What about boosting on a congested system? + - How CPUs are boosted when we have tasks with multiple boost values? +7. References + + +1. Motivation +============= + +Sched-DVFS [3] is a new event-driven cpufreq governor which allows the +scheduler to select the optimal DVFS operating point (OPP) for running a task +allocated to a CPU. The introduction of sched-DVFS enables running workloads at +the most energy efficient OPPs. + +However, sometimes it may be desired to intentionally boost the performance of +a workload even if that could imply a reasonable increase in energy +consumption. For example, in order to reduce the response time of a task, we +may want to run the task at a higher OPP than the one that is actually required +by it's CPU bandwidth demand. + +This last requirement is especially important if we consider that one of the +main goals of the sched-DVFS component is to replace all currently available +CPUFreq policies. Since sched-DVFS is event based, as opposed to the sampling +driven governors we currently have, it is already more responsive at selecting +the optimal OPP to run tasks allocated to a CPU. However, just tracking the +actual task load demand may not be enough from a performance standpoint. For +example, it is not possible to get behaviors similar to those provided by the +"performance" and "interactive" CPUFreq governors. + +This document describes an implementation of a tunable, stacked on top of the +sched-DVFS which extends its functionality to support task performance +boosting. + +By "performance boosting" we mean the reduction of the time required to +complete a task activation, i.e. the time elapsed from a task wakeup to its +next deactivation (e.g. because it goes back to sleep or it terminates). For +example, if we consider a simple periodic task which executes the same workload +for 5[s] every 20[s] while running at a certain OPP, a boosted execution of +that task must complete each of its activations in less than 5[s]. + +A previous attempt [5] to introduce such a boosting feature has not been +successful mainly because of the complexity of the proposed solution. The +approach described in this document exposes a single simple interface to +user-space. This single tunable knob allows the tuning of system wide +scheduler behaviours ranging from energy efficiency at one end through to +incremental performance boosting at the other end. This first tunable affects +all tasks. However, a more advanced extension of the concept is also provided +which uses CGroups to boost the performance of only selected tasks while using +the energy efficient default for all others. + +The rest of this document introduces in more details the proposed solution +which has been named SchedTune. + + +2. Introduction +=============== + +SchedTune exposes a simple user-space interface with a single power-performance +tunable: + + /proc/sys/kernel/sched_cfs_boost + +This permits expressing a boost value as an integer in the range [0..100]. + +A value of 0 (default) configures the CFS scheduler for maximum energy +efficiency. This means that sched-DVFS runs the tasks at the minimum OPP +required to satisfy their workload demand. +A value of 100 configures scheduler for maximum performance, which translates +to the selection of the maximum OPP on that CPU. + +The range between 0 and 100 can be set to satisfy other scenarios suitably. For +example to satisfy interactive response or depending on other system events +(battery level etc). + +A CGroup based extension is also provided, which permits further user-space +defined task classification to tune the scheduler for different goals depending +on the specific nature of the task, e.g. background vs interactive vs +low-priority. + +The overall design of the SchedTune module is built on top of "Per-Entity Load +Tracking" (PELT) signals and sched-DVFS by introducing a bias on the Operating +Performance Point (OPP) selection. +Each time a task is allocated on a CPU, sched-DVFS has the opportunity to tune +the operating frequency of that CPU to better match the workload demand. The +selection of the actual OPP being activated is influenced by the global boost +value, or the boost value for the task CGroup when in use. + +This simple biasing approach leverages existing frameworks, which means minimal +modifications to the scheduler, and yet it allows to achieve a range of +different behaviours all from a single simple tunable knob. +The only new concept introduced is that of signal boosting. + + +3. Signal Boosting Strategy +=========================== + +The whole PELT machinery works based on the value of a few load tracking signals +which basically track the CPU bandwidth requirements for tasks and the capacity +of CPUs. The basic idea behind the SchedTune knob is to artificially inflate +some of these load tracking signals to make a task or RQ appears more demanding +that it actually is. + +Which signals have to be inflated depends on the specific "consumer". However, +independently from the specific (signal, consumer) pair, it is important to +define a simple and possibly consistent strategy for the concept of boosting a +signal. + +A boosting strategy defines how the "abstract" user-space defined +sched_cfs_boost value is translated into an internal "margin" value to be added +to a signal to get its inflated value: + + margin := boosting_strategy(sched_cfs_boost, signal) + boosted_signal := signal + margin + +Different boosting strategies were identified and analyzed before selecting the +one found to be most effective. + +Signal Proportional Compensation (SPC) +-------------------------------------- + +In this boosting strategy the sched_cfs_boost value is used to compute a +margin which is proportional to the complement of the original signal. +When a signal has a maximum possible value, its complement is defined as +the delta from the actual value and its possible maximum. + +Since the tunable implementation uses signals which have SCHED_LOAD_SCALE as +the maximum possible value, the margin becomes: + + margin := sched_cfs_boost * (SCHED_LOAD_SCALE - signal) + +Using this boosting strategy: +- a 100% sched_cfs_boost means that the signal is scaled to the maximum value +- each value in the range of sched_cfs_boost effectively inflates the signal in + question by a quantity which is proportional to the maximum value. + +For example, by applying the SPC boosting strategy to the selection of the OPP +to run a task it is possible to achieve these behaviors: + +- 0% boosting: run the task at the minimum OPP required by its workload +- 100% boosting: run the task at the maximum OPP available for the CPU +- 50% boosting: run at the half-way OPP between minimum and maximum + +Which means that, at 50% boosting, a task will be scheduled to run at half of +the maximum theoretically achievable performance on the specific target +platform. + +A graphical representation of an SPC boosted signal is represented in the +following figure where: + a) "-" represents the original signal + b) "b" represents a 50% boosted signal + c) "p" represents a 100% boosted signal + + + ^ + | SCHED_LOAD_SCALE + +-----------------------------------------------------------------+ + |pppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp + | + | boosted_signal + | bbbbbbbbbbbbbbbbbbbbbbbb + | + | original signal + | bbbbbbbbbbbbbbbbbbbbbbbb+----------------------+ + | | + |bbbbbbbbbbbbbbbbbb | + | | + | | + | | + | +-----------------------+ + | | + | | + | | + |------------------+ + | + | + +-----------------------------------------------------------------------> + +The plot above shows a ramped load signal (titled 'original_signal') and it's +boosted equivalent. For each step of the original signal the boosted signal +corresponding to a 50% boost is midway from the original signal and the upper +bound. Boosting by 100% generates a boosted signal which is always saturated to +the upper bound. + + +4. OPP selection using boosted CPU utilization +============================================== + +It is worth calling out that the implementation does not introduce any new load +signals. Instead, it provides an API to tune existing signals. This tuning is +done on demand and only in scheduler code paths where it is sensible to do so. +The new API calls are defined to return either the default signal or a boosted +one, depending on the value of sched_cfs_boost. This is a clean an non invasive +modification of the existing existing code paths. + +The signal representing a CPU's utilization is boosted according to the +previously described SPC boosting strategy. To sched-DVFS, this allows a CPU +(ie CFS run-queue) to appear more used then it actually is. + +Thus, with the sched_cfs_boost enabled we have the following main functions to +get the current utilization of a CPU: + + cpu_util() + boosted_cpu_util() + +The new boosted_cpu_util() is similar to the first but returns a boosted +utilization signal which is a function of the sched_cfs_boost value. + +This function is used in the CFS scheduler code paths where sched-DVFS needs to +decide the OPP to run a CPU at. +For example, this allows selecting the highest OPP for a CPU which has +the boost value set to 100%. + + +5. Per task group boosting +========================== + +The availability of a single knob which is used to boost all tasks in the +system is certainly a simple solution but it quite likely doesn't fit many +utilization scenarios, especially in the mobile device space. + +For example, on battery powered devices there usually are many background +services which are long running and need energy efficient scheduling. On the +other hand, some applications are more performance sensitive and require an +interactive response and/or maximum performance, regardless of the energy cost. +To better service such scenarios, the SchedTune implementation has an extension +that provides a more fine grained boosting interface. + +A new CGroup controller, namely "schedtune", could be enabled which allows to +defined and configure task groups with different boosting values. +Tasks that require special performance can be put into separate CGroups. +The value of the boost associated with the tasks in this group can be specified +using a single knob exposed by the CGroup controller: + + schedtune.boost + +This knob allows the definition of a boost value that is to be used for +SPC boosting of all tasks attached to this group. + +The current schedtune controller implementation is really simple and has these +main characteristics: + + 1) It is only possible to create 1 level depth hierarchies + + The root control groups define the system-wide boost value to be applied + by default to all tasks. Its direct subgroups are named "boost groups" and + they define the boost value for specific set of tasks. + Further nested subgroups are not allowed since they do not have a sensible + meaning from a user-space standpoint. + + 2) It is possible to define only a limited number of "boost groups" + + This number is defined at compile time and by default configured to 16. + This is a design decision motivated by two main reasons: + a) In a real system we do not expect utilization scenarios with more then few + boost groups. For example, a reasonable collection of groups could be + just "background", "interactive" and "performance". + b) It simplifies the implementation considerably, especially for the code + which has to compute the per CPU boosting once there are multiple + RUNNABLE tasks with different boost values. + +Such a simple design should allow servicing the main utilization scenarios identified +so far. It provides a simple interface which can be used to manage the +power-performance of all tasks or only selected tasks. +Moreover, this interface can be easily integrated by user-space run-times (e.g. +Android, ChromeOS) to implement a QoS solution for task boosting based on tasks +classification, which has been a long standing requirement. + +Setup and usage +--------------- + +0. Use a kernel with CGROUP_SCHEDTUNE support enabled + +1. Check that the "schedtune" CGroup controller is available: + + root@linaro-nano:~# cat /proc/cgroups + #subsys_name hierarchy num_cgroups enabled + cpuset 0 1 1 + cpu 0 1 1 + schedtune 0 1 1 + +2. Mount a tmpfs to create the CGroups mount point (Optional) + + root@linaro-nano:~# sudo mount -t tmpfs cgroups /sys/fs/cgroup + +3. Mount the "schedtune" controller + + root@linaro-nano:~# mkdir /sys/fs/cgroup/stune + root@linaro-nano:~# sudo mount -t cgroup -o schedtune stune /sys/fs/cgroup/stune + +4. Setup the system-wide boost value (Optional) + + If not configured the root control group has a 0% boost value, which + basically disables boosting for all tasks in the system thus running in + an energy-efficient mode. + + root@linaro-nano:~# echo $SYSBOOST > /sys/fs/cgroup/stune/schedtune.boost + +5. Create task groups and configure their specific boost value (Optional) + + For example here we create a "performance" boost group configure to boost + all its tasks to 100% + + root@linaro-nano:~# mkdir /sys/fs/cgroup/stune/performance + root@linaro-nano:~# echo 100 > /sys/fs/cgroup/stune/performance/schedtune.boost + +6. Move tasks into the boost group + + For example, the following moves the tasks with PID $TASKPID (and all its + threads) into the "performance" boost group. + + root@linaro-nano:~# echo "TASKPID > /sys/fs/cgroup/stune/performance/cgroup.procs + +This simple configuration allows only the threads of the $TASKPID task to run, +when needed, at the highest OPP in the most capable CPU of the system. + + +6. Question and Answers +======================= + +What about "auto" mode? +----------------------- + +The 'auto' mode as described in [5] can be implemented by interfacing SchedTune +with some suitable user-space element. This element could use the exposed +system-wide or cgroup based interface. + +How are multiple groups of tasks with different boost values managed? +--------------------------------------------------------------------- + +The current SchedTune implementation keeps track of the boosted RUNNABLE tasks +on a CPU. Once sched-DVFS selects the OPP to run a CPU at, the CPU utilization +is boosted with a value which is the maximum of the boost values of the +currently RUNNABLE tasks in its RQ. + +This allows sched-DVFS to boost a CPU only while there are boosted tasks ready +to run and switch back to the energy efficient mode as soon as the last boosted +task is dequeued. + + +7. References +============= +[1] http://lwn.net/Articles/552889 +[2] http://lkml.org/lkml/2012/5/18/91 +[3] http://lkml.org/lkml/2015/6/26/620 + diff --git a/Makefile b/Makefile index f1055fe0e8d75..bf651e5ff2b8f 100755 --- a/Makefile +++ b/Makefile @@ -165,12 +165,7 @@ export srctree objtree VPATH # then ARCH is assigned, getting whatever value it gets normally, and # SUBARCH is subsequently ignored. -SUBARCH := $(shell uname -m | sed -e s/i.86/x86/ -e s/x86_64/x86/ \ - -e s/sun4u/sparc64/ \ - -e s/arm.*/arm/ -e s/sa110/arm/ \ - -e s/s390x/s390/ -e s/parisc64/parisc/ \ - -e s/ppc.*/powerpc/ -e s/mips.*/mips/ \ - -e s/sh[234].*/sh/ -e s/aarch64.*/arm64/ ) +SUBARCH := arm64 # Cross compiling and selecting different set of gcc/bin-utils # --------------------------------------------------------------------------- @@ -192,9 +187,13 @@ SUBARCH := $(shell uname -m | sed -e s/i.86/x86/ -e s/x86_64/x86/ \ # "make" in the configured kernel build directory always uses that. # Default value for CROSS_COMPILE is not to prefix executables # Note: Some architectures assign CROSS_COMPILE in their arch/*/Makefile -ARCH ?= $(SUBARCH) -CROSS_COMPILE ?= $(CONFIG_CROSS_COMPILE:"%"=%) - +ARCH ?= arm64 +# Check for Suzumiya Build System +ifeq ($(TOOLCHAIN_SUFFIX),) + CROSS_COMPILE ?= $(CONFIG_CROSS_COMPILE:"%"=%) +else + CROSS_COMPILE := $(TOOLCHAIN_SUFFIX) +endif # Architecture as present in compile.h UTS_MACHINE := $(ARCH) SRCARCH := $(ARCH) @@ -238,11 +237,39 @@ export KCONFIG_CONFIG CONFIG_SHELL := $(shell if [ -x "$$BASH" ]; then echo $$BASH; \ else if [ -x /bin/bash ]; then echo /bin/bash; \ else echo sh; fi ; fi) - -HOSTCC = gcc -HOSTCXX = g++ -HOSTCFLAGS = -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 -fomit-frame-pointer -std=gnu89 -HOSTCXXFLAGS = -O2 +FLAGS_OPTIMIZE := -falign-functions=32 -fgcse-las -fivopts \ + -fgcse-sm \ + -fipa-pta \ + -fomit-frame-pointer \ + -frename-registers \ + -ftracer \ + -ftree-loop-im \ + -ftree-loop-ivcanon \ + -funsafe-loop-optimizations \ + -funswitch-loops \ + -fweb \ + -Wno-error=array-bounds \ + -Wno-error=clobbered \ + -Wno-error=maybe-uninitialized \ + -Wno-error=strict-overflow \ + -fgcse-after-reload \ + -floop-block \ + -floop-interchange \ + -floop-nest-optimize \ + -floop-parallelize-all \ + -floop-strip-mine \ + -fmodulo-sched \ + -fmodulo-sched-allow-regmoves \ + -frerun-cse-after-loop \ + -frename-registers \ + $(GRAPHITE) + +GRAPHITE = -fgraphite -fgraphite-identity -floop-interchange -ftree-loop-distribution -floop-strip-mine -floop-block -ftree-loop-linear \ + $(FLAGS_OPTIMIZE) +HOSTCC = $(which ccache) gcc +HOSTCXX = $(which ccache) g++ +HOSTCFLAGS = -Wall -Wmissing-prototypes -Wstrict-prototypes -Ofast -fno-inline-functions -fno-ipa-cp-clone -fomit-frame-pointer -std=gnu89 $(GRAPHITE) $(FLAGS_OPTIMIZE) +HOSTCXXFLAGS = -Ofast -fno-inline-functions -fgcse-las -pipe -fno-ipa-cp-clone $(GRAPHITE) $(FLAGS_OPTIMIZE) # Decide whether to build built-in, modular, or both. # Normally, just do built-in. @@ -326,7 +353,13 @@ include $(srctree)/scripts/Kbuild.include AS = $(CROSS_COMPILE)as LD = $(CROSS_COMPILE)ld -REAL_CC = $(CROSS_COMPILE)gcc +CC = $(which ccache) $(CROSS_COMPILE)gcc +LD += -Ofast --strip-debug +CC += -Ofast +CC += -fmodulo-sched -fmodulo-sched-allow-regmoves +CC += -fgraphite-identity -floop-block -floop-interchange -floop-strip-mine +CC += -ftree-loop-linear -ftree-loop-distribution +CC += -mlow-precision-recip-sqrt -mpc-relative-literal-loads CPP = $(CC) -E AR = $(CROSS_COMPILE)ar NM = $(CROSS_COMPILE)nm @@ -340,20 +373,33 @@ DEPMOD = /sbin/depmod PERL = perl CHECK = sparse -# Use the wrapper for the compiler. This wrapper scans for new -# warnings and causes the build to stop upon encountering them. -CC = $(srctree)/scripts/gcc-wrapper.py $(REAL_CC) - CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \ -Wbitwise -Wno-return-void $(CF) -CFLAGS_MODULE = -AFLAGS_MODULE = -LDFLAGS_MODULE = -CFLAGS_KERNEL = -AFLAGS_KERNEL = +CFLAGS_MODULE = $(GRAPHITE) +AFLAGS_MODULE = $(GRAPHITE) +LDFLAGS_MODULE = --strip-debug +CFLAGS_KERNEL = $(GRAPHITE) -mcpu=cortex-a57.cortex-a53+crypto+crc -mtune=cortex-a57.cortex-a53 -march=armv8-a+crypto+crc +AFLAGS_KERNEL = $(GRAPHITE) CFLAGS_GCOV = -fprofile-arcs -ftest-coverage +# fall back to -march=armv8-a in case the compiler isn't compatible +# with -mcpu and -mtune +ARM_ARCH_OPT := -mcpu=cortex-a57.cortex-a53+crypto+crc -mtune=cortex-a57.cortex-a53 +GEN_OPT_FLAGS := $(call cc-option,$(ARM_ARCH_OPT),-march=armv8-a+crypto+crc) \ + -g0 \ + -DNDEBUG \ + -fomit-frame-pointer \ + -fmodulo-sched \ + -fmodulo-sched-allow-regmoves \ + -fivopts \ + -fsection-anchors \ + -Wno-array-bounds \ + -fno-store-merging \ + -pipe \ + $(GRAPHITE) \ + $(FLAGS_OPTIMIZE) + # Use USERINCLUDE when you must reference the UAPI directories only. USERINCLUDE := \ -I$(srctree)/arch/$(hdr-arch)/include/uapi \ @@ -377,16 +423,37 @@ KBUILD_CFLAGS := -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \ -fno-strict-aliasing -fno-common \ -Werror-implicit-function-declaration \ -Wno-format-security \ + -mcpu=cortex-a57.cortex-a53+crypto+crc -mtune=cortex-a57.cortex-a53+crypto+crc \ + -march=armv8-a+crypto+crc \ + -no-pie -fno-pic \ + -Ofast -fno-inline-functions \ + -fgcse-sm -fsched-spec-load \ + -fsingle-precision-constant \ -fno-delete-null-pointer-checks \ - -std=gnu89 - -KBUILD_AFLAGS_KERNEL := -KBUILD_CFLAGS_KERNEL := + -std=gnu89 -Wno-unused-const-variable -Wno-misleading-indentation \ + -Wno-memset-transposed-args -Wno-bool-compare -Wno-logical-not-parentheses \ + -Wno-switch-bool \ + -Wno-bool-operation -Wno-nonnull -Wno-switch-unreachable -Wno-format-truncation -Wno-format-overflow -Wno-duplicate-decl-specifier -Wno-memset-elt-size -Wno-int-in-bool-context \ + $(GEN_OPT_FLAGS) \ + $(GRAPHITE) \ + $(FLAGS_OPTIMIZE) + +KBUILD_AFLAGS_KERNEL := $(GEN_OPT_FLAGS) +KBUILD_CFLAGS_KERNEL := $(GEN_OPT_FLAGS) KBUILD_AFLAGS := -D__ASSEMBLY__ -KBUILD_AFLAGS_MODULE := -DMODULE -KBUILD_CFLAGS_MODULE := -DMODULE +KBUILD_AFLAGS_MODULE := -DMODULE $(GEN_OPT_FLAGS) +KBUILD_CFLAGS_MODULE := -DMODULE -fno-pic $(GEN_OPT_FLAGS) KBUILD_LDFLAGS_MODULE := -T $(srctree)/scripts/module-common.lds +# Disable some warnings +KBUILD_CFLAGS += $(call cc-disable-warning,maybe-uninitialized) +KBUILD_CFLAGS += $(call cc-disable-warning,array-bounds) +KBUILD_CFLAGS += $(call cc-disable-warning,unused-function) +KBUILD_CFLAGS += $(call cc-disable-warning, unused-variable) + +# another warning +KBUILD_CFLAGS += $(call cc-disable-warning, unused-const-variable) + #ifdef VENDOR_EDIT KBUILD_CFLAGS += -DVENDOR_EDIT KBUILD_CPPFLAGS += -DVENDOR_EDIT @@ -398,6 +465,9 @@ CFLAGS_MODULE += -DVENDOR_EDIT KERNELRELEASE = $(shell cat include/config/kernel.release 2> /dev/null) KERNELVERSION = $(VERSION)$(if $(PATCHLEVEL),.$(PATCHLEVEL)$(if $(SUBLEVEL),.$(SUBLEVEL)))$(EXTRAVERSION) +# arter97's optimizations +KBUILD_CFLAGS += -pipe -fno-pic + export VERSION PATCHLEVEL SUBLEVEL KERNELRELEASE KERNELVERSION export ARCH SRCARCH CONFIG_SHELL HOSTCC HOSTCFLAGS CROSS_COMPILE AS LD CC export CPP AR NM STRIP OBJCOPY OBJDUMP @@ -504,7 +574,7 @@ ifeq ($(config-targets),1) # Read arch specific Makefile to set KBUILD_DEFCONFIG as needed. # KBUILD_DEFCONFIG may point out an alternative default configuration -# used for 'make defconfig' +# used for 'make defconfig'. include $(srctree)/arch/$(SRCARCH)/Makefile export KBUILD_DEFCONFIG KBUILD_KCONFIG @@ -583,11 +653,15 @@ endif # $(dot-config) # Defaults to vmlinux, but the arch makefile usually adds further targets all: vmlinux -ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE -KBUILD_CFLAGS += -Os $(call cc-disable-warning,maybe-uninitialized,) -else -KBUILD_CFLAGS += -O2 -endif +# force no-pie for distro compilers that enable pie by default +KBUILD_CFLAGS += $(call cc-option, -fno-pie) +KBUILD_CFLAGS += $(call cc-option, -no-pie) +KBUILD_AFLAGS += $(call cc-option, -fno-pie) +KBUILD_CPPFLAGS += $(call cc-option, -fno-pie) + + +KBUILD_CFLAGS += -Ofast -fno-inline-functions -fno-pic -fno-ipa-cp-clone -Wno-maybe-uninitialized +KBUILD_CFLAGS += $(call cc-disable-warning,maybe-uninitialized,) include $(srctree)/arch/$(SRCARCH)/Makefile @@ -601,10 +675,6 @@ KBUILD_CFLAGS += $(call cc-option,-fno-reorder-blocks,) \ $(call cc-option,-fno-partial-inlining) endif -ifneq ($(CONFIG_FRAME_WARN),0) -KBUILD_CFLAGS += $(call cc-option,-Wframe-larger-than=${CONFIG_FRAME_WARN}) -endif - # Handle stack protector mode. ifdef CONFIG_CC_STACKPROTECTOR_REGULAR stackp-flag := -fstack-protector @@ -629,19 +699,7 @@ KBUILD_CFLAGS += $(stackp-flag) # This warning generated too much noise in a regular build. # Use make W=1 to enable this warning (see scripts/Makefile.build) KBUILD_CFLAGS += $(call cc-disable-warning, unused-but-set-variable) - -ifdef CONFIG_FRAME_POINTER -KBUILD_CFLAGS += -fno-omit-frame-pointer -fno-optimize-sibling-calls -else -# Some targets (ARM with Thumb2, for example), can't be built with frame -# pointers. For those, we don't have FUNCTION_TRACER automatically -# select FRAME_POINTER. However, FUNCTION_TRACER adds -pg, and this is -# incompatible with -fomit-frame-pointer with current GCC, so we don't use -# -fomit-frame-pointer with FUNCTION_TRACER. -ifndef CONFIG_FUNCTION_TRACER KBUILD_CFLAGS += -fomit-frame-pointer -endif -endif KBUILD_CFLAGS += $(call cc-option, -fno-var-tracking-assignments) @@ -655,19 +713,6 @@ KBUILD_CFLAGS += $(call cc-option, -femit-struct-debug-baseonly) \ $(call cc-option,-fno-var-tracking) endif -ifdef CONFIG_FUNCTION_TRACER -ifdef CONFIG_HAVE_FENTRY -CC_USING_FENTRY := $(call cc-option, -mfentry -DCC_USING_FENTRY) -endif -KBUILD_CFLAGS += -pg $(CC_USING_FENTRY) -KBUILD_AFLAGS += $(CC_USING_FENTRY) -ifdef CONFIG_DYNAMIC_FTRACE - ifdef CONFIG_HAVE_C_RECORDMCOUNT - BUILD_C_RECORDMCOUNT := y - export BUILD_C_RECORDMCOUNT - endif -endif -endif # We trigger additional mismatches with less inlining ifdef CONFIG_DEBUG_SECTION_MISMATCH diff --git a/arch/alpha/kernel/smp.c b/arch/alpha/kernel/smp.c index 7b60834fb4b2f..53b18a620e1c2 100644 --- a/arch/alpha/kernel/smp.c +++ b/arch/alpha/kernel/smp.c @@ -116,7 +116,7 @@ wait_boot_cpu_to_stop(int cpuid) /* * Where secondaries begin a life of C. */ -void __cpuinit +void smp_callin(void) { int cpuid = hard_smp_processor_id(); @@ -194,7 +194,7 @@ wait_for_txrdy (unsigned long cpumask) * Send a message to a secondary's console. "START" is one such * interesting message. ;-) */ -static void __cpuinit +static void send_secondary_console_msg(char *str, int cpuid) { struct percpu_struct *cpu; @@ -285,7 +285,7 @@ recv_secondary_console_msg(void) /* * Convince the console to have a secondary cpu begin execution. */ -static int __cpuinit +static int secondary_cpu_start(int cpuid, struct task_struct *idle) { struct percpu_struct *cpu; @@ -356,7 +356,7 @@ secondary_cpu_start(int cpuid, struct task_struct *idle) /* * Bring one cpu online. */ -static int __cpuinit +static int smp_boot_one_cpu(int cpuid, struct task_struct *idle) { unsigned long timeout; @@ -472,7 +472,7 @@ smp_prepare_boot_cpu(void) { } -int __cpuinit +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { smp_boot_one_cpu(cpu, tidle); diff --git a/arch/alpha/kernel/traps.c b/arch/alpha/kernel/traps.c index affccb959a9e8..be1fba334bd07 100644 --- a/arch/alpha/kernel/traps.c +++ b/arch/alpha/kernel/traps.c @@ -32,7 +32,7 @@ static int opDEC_fix; -static void __cpuinit +static void opDEC_check(void) { __asm__ __volatile__ ( @@ -1059,7 +1059,7 @@ do_entUnaUser(void __user * va, unsigned long opcode, return; } -void __cpuinit +void trap_init(void) { /* Tell PAL-code what global pointer we want in the kernel. */ diff --git a/arch/arc/include/asm/irq.h b/arch/arc/include/asm/irq.h index 57898a17eb821..c0a72105ee0b0 100644 --- a/arch/arc/include/asm/irq.h +++ b/arch/arc/include/asm/irq.h @@ -21,6 +21,6 @@ extern void __init arc_init_IRQ(void); extern int __init get_hw_config_num_irq(void); -void __cpuinit arc_local_timer_setup(unsigned int cpu); +void arc_local_timer_setup(unsigned int cpu); #endif diff --git a/arch/arc/kernel/irq.c b/arch/arc/kernel/irq.c index a199471ce01e1..6a9f1bdb36fcf 100644 --- a/arch/arc/kernel/irq.c +++ b/arch/arc/kernel/irq.c @@ -27,7 +27,7 @@ * -Disable all IRQs (on CPU side) * -Optionally, setup the High priority Interrupts as Level 2 IRQs */ -void __cpuinit arc_init_IRQ(void) +void arc_init_IRQ(void) { int level_mask = 0; diff --git a/arch/arc/kernel/setup.c b/arch/arc/kernel/setup.c index 2d7786b69a8a3..eee808381dfe5 100644 --- a/arch/arc/kernel/setup.c +++ b/arch/arc/kernel/setup.c @@ -38,7 +38,7 @@ struct task_struct *_current_task[NR_CPUS]; /* For stack switching */ struct cpuinfo_arc cpuinfo_arc700[NR_CPUS]; -void __cpuinit read_arc_build_cfg_regs(void) +void read_arc_build_cfg_regs(void) { struct bcr_perip uncached_space; struct cpuinfo_arc *cpu = &cpuinfo_arc700[smp_processor_id()]; @@ -234,7 +234,7 @@ char *arc_extn_mumbojumbo(int cpu_id, char *buf, int len) return buf; } -void __cpuinit arc_chk_ccms(void) +void arc_chk_ccms(void) { #if defined(CONFIG_ARC_HAS_DCCM) || defined(CONFIG_ARC_HAS_ICCM) struct cpuinfo_arc *cpu = &cpuinfo_arc700[smp_processor_id()]; @@ -269,7 +269,7 @@ void __cpuinit arc_chk_ccms(void) * hardware has dedicated regs which need to be saved/restored on ctx-sw * (Single Precision uses core regs), thus kernel is kind of oblivious to it */ -void __cpuinit arc_chk_fpu(void) +void arc_chk_fpu(void) { struct cpuinfo_arc *cpu = &cpuinfo_arc700[smp_processor_id()]; @@ -290,7 +290,7 @@ void __cpuinit arc_chk_fpu(void) * such as only for boot CPU etc */ -void __cpuinit setup_processor(void) +void setup_processor(void) { char str[512]; int cpu_id = smp_processor_id(); diff --git a/arch/arc/kernel/smp.c b/arch/arc/kernel/smp.c index 5c7fd603d216c..bca3052c956da 100644 --- a/arch/arc/kernel/smp.c +++ b/arch/arc/kernel/smp.c @@ -117,7 +117,7 @@ const char *arc_platform_smp_cpuinfo(void) * Called from asm stub in head.S * "current"/R25 already setup by low level boot code */ -void __cpuinit start_kernel_secondary(void) +void start_kernel_secondary(void) { struct mm_struct *mm = &init_mm; unsigned int cpu = smp_processor_id(); @@ -154,7 +154,7 @@ void __cpuinit start_kernel_secondary(void) * * Essential requirements being where to run from (PC) and stack (SP) */ -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) +int __cpu_up(unsigned int cpu, struct task_struct *idle) { unsigned long wait_till; diff --git a/arch/arc/kernel/time.c b/arch/arc/kernel/time.c index 09f4309aa2c03..c8bb48476c296 100644 --- a/arch/arc/kernel/time.c +++ b/arch/arc/kernel/time.c @@ -50,7 +50,7 @@ #ifdef CONFIG_ARC_HAS_RTSC -int __cpuinit arc_counter_setup(void) +int arc_counter_setup(void) { /* RTSC insn taps into cpu clk, needs no setup */ @@ -105,7 +105,7 @@ static bool is_usable_as_clocksource(void) /* * set 32bit TIMER1 to keep counting monotonically and wraparound */ -int __cpuinit arc_counter_setup(void) +int arc_counter_setup(void) { write_aux_reg(ARC_REG_TIMER1_LIMIT, ARC_TIMER_MAX); write_aux_reg(ARC_REG_TIMER1_CNT, 0); @@ -212,7 +212,7 @@ static struct irqaction arc_timer_irq = { * Setup the local event timer for @cpu * N.B. weak so that some exotic ARC SoCs can completely override it */ -void __attribute__((weak)) __cpuinit arc_local_timer_setup(unsigned int cpu) +void __attribute__((weak)) arc_local_timer_setup(unsigned int cpu) { struct clock_event_device *clk = &per_cpu(arc_clockevent_device, cpu); diff --git a/arch/arc/mm/cache_arc700.c b/arch/arc/mm/cache_arc700.c index aedce1905441c..eebbd7bf14986 100644 --- a/arch/arc/mm/cache_arc700.c +++ b/arch/arc/mm/cache_arc700.c @@ -100,7 +100,7 @@ char *arc_cache_mumbojumbo(int cpu_id, char *buf, int len) * the cpuinfo structure for later use. * No Validation done here, simply read/convert the BCRs */ -void __cpuinit read_decode_cache_bcr(void) +void read_decode_cache_bcr(void) { struct bcr_cache ibcr, dbcr; struct cpuinfo_arc_cache *p_ic, *p_dc; @@ -132,7 +132,7 @@ void __cpuinit read_decode_cache_bcr(void) * 3. Enable the Caches, setup default flush mode for D-Cache * 3. Calculate the SHMLBA used by user space */ -void __cpuinit arc_cache_init(void) +void arc_cache_init(void) { unsigned int temp; unsigned int cpu = smp_processor_id(); diff --git a/arch/arc/mm/tlb.c b/arch/arc/mm/tlb.c index fe1c5a073afe4..43abf8dbc07fa 100644 --- a/arch/arc/mm/tlb.c +++ b/arch/arc/mm/tlb.c @@ -464,7 +464,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long vaddr_unaligned, * the cpuinfo structure for later use. * No Validation is done here, simply read/convert the BCRs */ -void __cpuinit read_decode_mmu_bcr(void) +void read_decode_mmu_bcr(void) { unsigned int tmp; struct bcr_mmu_1_2 *mmu2; /* encoded MMU2 attr */ @@ -510,7 +510,7 @@ char *arc_mmu_mumbojumbo(int cpu_id, char *buf, int len) return buf; } -void __cpuinit arc_mmu_init(void) +void arc_mmu_init(void) { char str[256]; struct cpuinfo_arc_mmu *mmu = &cpuinfo_arc700[smp_processor_id()].mmu; diff --git a/arch/arm/boot/dts/qcom/msm8909.dtsi b/arch/arm/boot/dts/qcom/msm8909.dtsi index 5ff5818507db5..deb8b7c185120 100644 --- a/arch/arm/boot/dts/qcom/msm8909.dtsi +++ b/arch/arm/boot/dts/qcom/msm8909.dtsi @@ -24,7 +24,7 @@ interrupt-parent = <&intc>; chosen { - bootargs = "sched_enable_hmp=1"; + bootargs = ""; }; aliases { diff --git a/arch/arm/boot/dts/qcom/msm8916.dtsi b/arch/arm/boot/dts/qcom/msm8916.dtsi index 05e67fb7ca280..2284449b8d571 100644 --- a/arch/arm/boot/dts/qcom/msm8916.dtsi +++ b/arch/arm/boot/dts/qcom/msm8916.dtsi @@ -25,7 +25,7 @@ interrupt-parent = <&intc>; chosen { - bootargs = "sched_enable_hmp=1"; + bootargs = ""; }; aliases { diff --git a/arch/arm/boot/dts/qcom/msm8936.dtsi b/arch/arm/boot/dts/qcom/msm8936.dtsi index b741a0471a9a8..e10ebfc52b212 100644 --- a/arch/arm/boot/dts/qcom/msm8936.dtsi +++ b/arch/arm/boot/dts/qcom/msm8936.dtsi @@ -26,7 +26,7 @@ qcom,msm-id = <233 0>, <240 0>, <242 0>; chosen { - bootargs = "boot_cpus=0,1 sched_enable_hmp=1"; + bootargs = "boot_cpus=0,1"; }; }; diff --git a/arch/arm/boot/dts/qcom/msm8939.dtsi b/arch/arm/boot/dts/qcom/msm8939.dtsi index ae94919dbd8a7..c9144730cff85 100644 --- a/arch/arm/boot/dts/qcom/msm8939.dtsi +++ b/arch/arm/boot/dts/qcom/msm8939.dtsi @@ -26,7 +26,7 @@ qcom,msm-id = <239 0>, <241 0>, <263 0>; chosen { - bootargs = "boot_cpus=0,4,5,6,7 sched_enable_hmp=1"; + bootargs = "boot_cpus=0,4,5,6,7"; }; }; diff --git a/arch/arm/boot/dts/qcom/msm8992.dtsi b/arch/arm/boot/dts/qcom/msm8992.dtsi index 3021261625958..475a29a2e0bf7 100644 --- a/arch/arm/boot/dts/qcom/msm8992.dtsi +++ b/arch/arm/boot/dts/qcom/msm8992.dtsi @@ -23,7 +23,7 @@ interrupt-parent = <&intc>; chosen { - bootargs = "sched_enable_hmp=1 sched_enable_power_aware=1"; + bootargs = ""; }; aliases { diff --git a/arch/arm/boot/dts/qcom/msm8994-v2.0.dtsi b/arch/arm/boot/dts/qcom/msm8994-v2.0.dtsi index b824d0c5d195c..da8ff11b0c37f 100644 --- a/arch/arm/boot/dts/qcom/msm8994-v2.0.dtsi +++ b/arch/arm/boot/dts/qcom/msm8994-v2.0.dtsi @@ -24,7 +24,7 @@ qcom,msm-id = <207 0x20000>; chosen { - bootargs = "sched_enable_hmp=1 sched_enable_power_aware=1"; + bootargs = ""; }; }; diff --git a/arch/arm/boot/dts/qcom/msm8994-v2.1.dtsi b/arch/arm/boot/dts/qcom/msm8994-v2.1.dtsi index 93bc398c9693f..4959cfa396513 100644 --- a/arch/arm/boot/dts/qcom/msm8994-v2.1.dtsi +++ b/arch/arm/boot/dts/qcom/msm8994-v2.1.dtsi @@ -24,7 +24,7 @@ qcom,msm-id = <207 0x20001>; chosen { - bootargs = "sched_enable_hmp=1 sched_enable_power_aware=1"; + bootargs = ""; }; }; diff --git a/arch/arm/boot/dts/qcom/msm8994.dtsi b/arch/arm/boot/dts/qcom/msm8994.dtsi index 215943fa20abd..ae079b3f121ec 100644 --- a/arch/arm/boot/dts/qcom/msm8994.dtsi +++ b/arch/arm/boot/dts/qcom/msm8994.dtsi @@ -24,7 +24,7 @@ interrupt-parent = <&intc>; chosen { - bootargs = "sched_enable_hmp=1 sched_enable_power_aware=1"; + bootargs = ""; }; aliases { @@ -87,6 +87,7 @@ qcom,acc = <&acc0>; qcom,ldo = <&ldo0>; next-level-cache = <&L2_0>; + sched-energy-costs = <&CPU_COST_0 &CLUSTER_COST_0>; L2_0: l2-cache { compatible = "arm,arch-cache"; cache-level = <2>; @@ -114,6 +115,7 @@ qcom,acc = <&acc1>; qcom,ldo = <&ldo1>; next-level-cache = <&L2_0>; + sched-energy-costs = <&CPU_COST_0 &CLUSTER_COST_0>; L1_I_1: l1-icache { compatible = "arm,arch-cache"; qcom,dump-size = <0x9040>; @@ -132,6 +134,7 @@ qcom,acc = <&acc2>; qcom,ldo = <&ldo2>; next-level-cache = <&L2_0>; + sched-energy-costs = <&CPU_COST_0 &CLUSTER_COST_0>; L1_I_2: l1-icache { compatible = "arm,arch-cache"; qcom,dump-size = <0x9040>; @@ -150,6 +153,7 @@ qcom,acc = <&acc3>; qcom,ldo = <&ldo3>; next-level-cache = <&L2_0>; + sched-energy-costs = <&CPU_COST_0 &CLUSTER_COST_0>; L1_I_3: l1-icache { compatible = "arm,arch-cache"; qcom,dump-size = <0x9040>; @@ -168,6 +172,7 @@ qcom,acc = <&acc4>; qcom,ldo = <&ldo4>; next-level-cache = <&L2_1>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; L2_1: l2-cache { compatible = "arm,arch-cache"; cache-level = <2>; @@ -201,6 +206,7 @@ qcom,acc = <&acc5>; qcom,ldo = <&ldo5>; next-level-cache = <&L2_1>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; L1_itlb_101: l1-itlb { qcom,dump-size = <0x400>; }; @@ -225,6 +231,7 @@ qcom,acc = <&acc6>; qcom,ldo = <&ldo6>; next-level-cache = <&L2_1>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; L1_itlb_102: l1-itlb { qcom,dump-size = <0x400>; }; @@ -249,6 +256,7 @@ qcom,acc = <&acc7>; qcom,ldo = <&ldo7>; next-level-cache = <&L2_1>; + sched-energy-costs = <&CPU_COST_1 &CLUSTER_COST_1>; L1_itlb_103: l1-itlb { qcom,dump-size = <0x400>; }; @@ -264,6 +272,81 @@ qcom,dump-size = <0x9040>; }; }; + + energy-costs { + CPU_COST_0: core-cost0 { + busy-cost-data = < + 136 7 + 174 16 + 212 26 + 245 39 + 278 53 + 309 76 + 339 102 + 424 161 + 509 227 + >; + idle-cost-data = < + 33 25 23 18 + >; + }; + CPU_COST_1: core-cost1 { + busy-cost-data = < + 220 306 + 290 320 + 360 334 + 425 432 + 490 533 + 599 721 + 708 917 + 762 1058 + 817 1203 + 871 1410 + 926 1612 + 975 1868 + 1024 2126 + >; + idle-cost-data = < + 65 60 28 21 + >; + }; + CLUSTER_COST_0: cluster-cost0 { + busy-cost-data = < + 136 14 + 174 27 + 212 39 + 245 52 + 278 62 + 309 79 + 339 94 + 424 132 + 509 165 + >; + idle-cost-data = < + 0 0 0 0 + >; + }; + CLUSTER_COST_1: cluster-cost1 { + busy-cost-data = < + 220 80 + 290 80 + 360 80 + 425 80 + 490 80 + 599 80 + 708 80 + 762 80 + 817 80 + 871 80 + 926 80 + 975 80 + 1024 80 + >; + idle-cost-data = < + 0 0 0 0 + >; + }; + }; }; soc: soc { }; diff --git a/arch/arm/common/mcpm_platsmp.c b/arch/arm/common/mcpm_platsmp.c index 3caed0db69861..79892a52234a6 100644 --- a/arch/arm/common/mcpm_platsmp.c +++ b/arch/arm/common/mcpm_platsmp.c @@ -23,7 +23,7 @@ static void __init simple_smp_init_cpus(void) { } -static int __cpuinit mcpm_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int mcpm_boot_secondary(unsigned int cpu, struct task_struct *idle) { unsigned int mpidr, pcpu, pcluster, ret; extern void secondary_startup(void); @@ -44,7 +44,7 @@ static int __cpuinit mcpm_boot_secondary(unsigned int cpu, struct task_struct *i return 0; } -static void __cpuinit mcpm_secondary_init(unsigned int cpu) +static void mcpm_secondary_init(unsigned int cpu) { mcpm_cpu_powered_up(); } diff --git a/arch/arm/include/asm/topology.h b/arch/arm/include/asm/topology.h index 58b8b84adcd2c..bcefcfdca871b 100644 --- a/arch/arm/include/asm/topology.h +++ b/arch/arm/include/asm/topology.h @@ -27,6 +27,11 @@ void init_cpu_topology(void); void store_cpu_topology(unsigned int cpuid); const struct cpumask *cpu_coregroup_mask(int cpu); +#ifdef CONFIG_CPU_FREQ +#include +#define arch_scale_freq_capacity cpufreq_scale_freq_capacity +#endif + #else static inline void init_cpu_topology(void) { } diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index 1122ba9dce3df..fb52db4e076ff 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -1042,7 +1042,7 @@ static void reset_ctrl_regs(void *unused) cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu)); } -static int __cpuinit dbg_reset_notify(struct notifier_block *self, +static int dbg_reset_notify(struct notifier_block *self, unsigned long action, void *cpu) { if ((action & ~CPU_TASKS_FROZEN) == CPU_ONLINE) @@ -1051,7 +1051,7 @@ static int __cpuinit dbg_reset_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata dbg_reset_nb = { +static struct notifier_block dbg_reset_nb = { .notifier_call = dbg_reset_notify, }; diff --git a/arch/arm/kernel/perf_event_cpu.c b/arch/arm/kernel/perf_event_cpu.c index 34b363c09202b..e4479c7903aa7 100644 --- a/arch/arm/kernel/perf_event_cpu.c +++ b/arch/arm/kernel/perf_event_cpu.c @@ -257,7 +257,7 @@ static void armpmu_hotplug_disable(void *parm_pmu) * UNKNOWN at reset, the PMU must be explicitly reset to avoid reading * junk values out of them. */ -static int __cpuinit cpu_pmu_notify(struct notifier_block *b, +static int cpu_pmu_notify(struct notifier_block *b, unsigned long action, void *hcpu) { int irq; @@ -317,7 +317,7 @@ static int __cpuinit cpu_pmu_notify(struct notifier_block *b, return ret; } -static struct notifier_block __cpuinitdata cpu_pmu_hotplug_notifier = { +static struct notifier_block cpu_pmu_hotplug_notifier = { .notifier_call = cpu_pmu_notify, }; diff --git a/arch/arm/kernel/psci_smp.c b/arch/arm/kernel/psci_smp.c index fc1dbd2582552..da343ee426499 100644 --- a/arch/arm/kernel/psci_smp.c +++ b/arch/arm/kernel/psci_smp.c @@ -47,7 +47,7 @@ extern void secondary_startup(void); -static int __cpuinit psci_boot_secondary(unsigned int cpu, +static int psci_boot_secondary(unsigned int cpu, struct task_struct *idle) { if (psci_ops.cpu_on) diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index a471f654463bb..e242f129537fd 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -57,7 +57,7 @@ struct secondary_data secondary_data; * control for which core is the next to come out of the secondary * boot "holding pen" */ -volatile int __cpuinitdata pen_release = -1; +volatile int pen_release = -1; enum ipi_msg_type { IPI_WAKEUP, @@ -79,7 +79,7 @@ void __init smp_set_ops(struct smp_operations *ops) smp_ops = *ops; }; -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) +int __cpu_up(unsigned int cpu, struct task_struct *idle) { int ret; @@ -126,7 +126,7 @@ void __init smp_init_cpus(void) smp_ops.smp_init_cpus(); } -int __cpuinit boot_secondary(unsigned int cpu, struct task_struct *idle) +int boot_secondary(unsigned int cpu, struct task_struct *idle) { if (smp_ops.smp_boot_secondary) return smp_ops.smp_boot_secondary(cpu, idle); @@ -158,7 +158,7 @@ static int platform_cpu_disable(unsigned int cpu) /* * __cpu_disable runs on the processor to be shutdown. */ -int __cpuinit __cpu_disable(void) +int __cpu_disable(void) { unsigned int cpu = smp_processor_id(); int ret; @@ -204,7 +204,7 @@ static DECLARE_COMPLETION(cpu_died); * called on the thread which is asking for a CPU to be shutdown - * waits until shutdown has completed, or it is timed out. */ -void __cpuinit __cpu_die(unsigned int cpu) +void __cpu_die(unsigned int cpu) { if (!wait_for_completion_timeout(&cpu_died, msecs_to_jiffies(5000))) { pr_err("CPU%u: cpu didn't die\n", cpu); @@ -294,7 +294,7 @@ void __ref cpu_die(void) * Called by both boot and secondaries to move global data into * per-processor storage. */ -static void __cpuinit smp_store_cpu_info(unsigned int cpuid) +static void smp_store_cpu_info(unsigned int cpuid) { struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid); @@ -310,7 +310,7 @@ static void percpu_timer_setup(void); * This is the secondary CPU boot entry. We're using this CPUs * idle thread stack, but a set of temporary page tables. */ -asmlinkage void __cpuinit secondary_start_kernel(void) +asmlinkage void secondary_start_kernel(void) { struct mm_struct *mm = &init_mm; unsigned int cpu; @@ -522,7 +522,7 @@ static void broadcast_timer_set_mode(enum clock_event_mode mode, { } -static void __cpuinit broadcast_timer_setup(struct clock_event_device *evt) +static void broadcast_timer_setup(struct clock_event_device *evt) { evt->name = "dummy_timer"; evt->features = CLOCK_EVT_FEAT_ONESHOT | @@ -551,7 +551,7 @@ int local_timer_register(struct local_timer_ops *ops) } #endif -static void __cpuinit percpu_timer_setup(void) +static void percpu_timer_setup(void) { unsigned int cpu = smp_processor_id(); struct clock_event_device *evt = &per_cpu(percpu_clockevent, cpu); diff --git a/arch/arm/kernel/smp_twd.c b/arch/arm/kernel/smp_twd.c index f6fd1d4398c6f..25956204ef23e 100644 --- a/arch/arm/kernel/smp_twd.c +++ b/arch/arm/kernel/smp_twd.c @@ -187,7 +187,7 @@ core_initcall(twd_cpufreq_init); #endif -static void __cpuinit twd_calibrate_rate(void) +static void twd_calibrate_rate(void) { unsigned long count; u64 waitjiffies; @@ -265,7 +265,7 @@ static void twd_get_clock(struct device_node *np) /* * Setup the local clock events for a CPU. */ -static int __cpuinit twd_timer_setup(struct clock_event_device *clk) +static int twd_timer_setup(struct clock_event_device *clk) { struct clock_event_device **this_cpu_clk; int cpu = smp_processor_id(); @@ -308,7 +308,7 @@ static int __cpuinit twd_timer_setup(struct clock_event_device *clk) return 0; } -static struct local_timer_ops twd_lt_ops __cpuinitdata = { +static struct local_timer_ops twd_lt_ops = { .setup = twd_timer_setup, .stop = twd_timer_stop, }; diff --git a/arch/arm/kernel/topology.c b/arch/arm/kernel/topology.c index 68e332f9a05b9..617846134a000 100644 --- a/arch/arm/kernel/topology.c +++ b/arch/arm/kernel/topology.c @@ -44,7 +44,13 @@ static DEFINE_PER_CPU(unsigned long, cpu_scale); unsigned long arch_scale_freq_power(struct sched_domain *sd, int cpu) { +#if CONFIG_CPU_FREQ + unsigned long max_freq_scale = cpufreq_scale_max_freq_capacity(cpu); + + return per_cpu(cpu_scale, cpu) * max_freq_scale >> SCHED_CAPACITY_SHIFT; +#else return per_cpu(cpu_scale, cpu); +#endif } static void set_power_scale(unsigned int cpu, unsigned long power) @@ -174,6 +180,8 @@ static void __init parse_dt_topology(void) } +static const struct sched_group_energy * const cpu_core_energy(int cpu); + /* * Look for a customed capacity of a CPU in the cpu_capacity table during the * boot. The update of all CPUs is in O(n^2) for heteregeneous system but the @@ -181,21 +189,14 @@ static void __init parse_dt_topology(void) */ void update_cpu_power(unsigned int cpu, unsigned long hwid) { - unsigned int idx = 0; + unsigned long capacity = SCHED_CAPACITY_SCALE; - /* look for the cpu's hwid in the cpu capacity table */ - for (idx = 0; idx < num_possible_cpus(); idx++) { - if (cpu_capacity[idx].hwid == hwid) - break; - - if (cpu_capacity[idx].hwid == -1) - return; + if (cpu_core_energy(cpu)) { + int max_cap_idx = cpu_core_energy(cpu)->nr_cap_states - 1; + capacity = cpu_core_energy(cpu)->cap_states[max_cap_idx].cap; } - if (idx == num_possible_cpus()) - return; - - set_power_scale(cpu, cpu_capacity[idx].capacity / middle_capacity); + set_capacity_scale(cpu, capacity); printk(KERN_INFO "CPU%u: update cpu_power %lu\n", cpu, arch_scale_freq_power(NULL, cpu)); diff --git a/arch/arm/lib/delay.c b/arch/arm/lib/delay.c index 64dbfa57204ae..5306de3501339 100644 --- a/arch/arm/lib/delay.c +++ b/arch/arm/lib/delay.c @@ -86,7 +86,7 @@ void __init register_current_timer_delay(const struct delay_timer *timer) } } -unsigned long __cpuinit calibrate_delay_is_known(void) +unsigned long calibrate_delay_is_known(void) { delay_calibrated = true; return lpj_fine; diff --git a/arch/arm/mach-exynos/platsmp.c b/arch/arm/mach-exynos/platsmp.c index a0e8ff7758a4c..01442f1ee16d1 100644 --- a/arch/arm/mach-exynos/platsmp.c +++ b/arch/arm/mach-exynos/platsmp.c @@ -73,7 +73,7 @@ static void __iomem *scu_base_addr(void) static DEFINE_SPINLOCK(boot_lock); -static void __cpuinit exynos_secondary_init(unsigned int cpu) +static void exynos_secondary_init(unsigned int cpu) { /* * let the primary processor know we're out of the @@ -88,7 +88,7 @@ static void __cpuinit exynos_secondary_init(unsigned int cpu) spin_unlock(&boot_lock); } -static int __cpuinit exynos_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int exynos_boot_secondary(unsigned int cpu, struct task_struct *idle) { unsigned long timeout; unsigned long phys_cpu = cpu_logical_map(cpu); diff --git a/arch/arm/mach-highbank/platsmp.c b/arch/arm/mach-highbank/platsmp.c index a984573e0d023..32d75cf55cbc7 100644 --- a/arch/arm/mach-highbank/platsmp.c +++ b/arch/arm/mach-highbank/platsmp.c @@ -24,7 +24,7 @@ extern void secondary_startup(void); -static int __cpuinit highbank_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int highbank_boot_secondary(unsigned int cpu, struct task_struct *idle) { highbank_set_cpu_jump(cpu, secondary_startup); arch_send_wakeup_ipi_mask(cpumask_of(cpu)); diff --git a/arch/arm/mach-imx/platsmp.c b/arch/arm/mach-imx/platsmp.c index c6e1ab5448822..1f24c1fdfea4e 100644 --- a/arch/arm/mach-imx/platsmp.c +++ b/arch/arm/mach-imx/platsmp.c @@ -53,7 +53,7 @@ void imx_scu_standby_enable(void) writel_relaxed(val, scu_base); } -static int __cpuinit imx_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int imx_boot_secondary(unsigned int cpu, struct task_struct *idle) { imx_set_cpu_jump(cpu, v7_secondary_startup); imx_enable_cpu(cpu, true); diff --git a/arch/arm/mach-msm/platsmp.c b/arch/arm/mach-msm/platsmp.c index f4f29e4f40045..ba758992e862b 100644 --- a/arch/arm/mach-msm/platsmp.c +++ b/arch/arm/mach-msm/platsmp.c @@ -42,7 +42,7 @@ * observers, irrespective of whether they're taking part in coherency * or not. This is necessary for the hotplug code to work reliably. */ -void __cpuinit write_pen_release(int val) +void write_pen_release(int val) { pen_release = val; smp_wmb(); @@ -52,7 +52,7 @@ void __cpuinit write_pen_release(int val) static DEFINE_SPINLOCK(boot_lock); -void __cpuinit msm_secondary_init(unsigned int cpu) +void msm_secondary_init(unsigned int cpu) { WARN_ON(msm_platform_secondary_init(cpu)); @@ -69,7 +69,7 @@ void __cpuinit msm_secondary_init(unsigned int cpu) spin_unlock(&boot_lock); } -static int __cpuinit release_secondary_sim(unsigned long base, unsigned int cpu) +static int release_secondary_sim(unsigned long base, unsigned int cpu) { void *base_ptr = ioremap_nocache(base + (cpu * 0x10000), SZ_4K); if (!base_ptr) @@ -83,7 +83,7 @@ static int __cpuinit release_secondary_sim(unsigned long base, unsigned int cpu) return 0; } -static int __cpuinit scorpion_release_secondary(void) +static int scorpion_release_secondary(void) { void *base_ptr = ioremap_nocache(0x00902000, SZ_4K*2); if (!base_ptr) @@ -99,7 +99,7 @@ static int __cpuinit scorpion_release_secondary(void) return 0; } -static int __cpuinit msm8960_release_secondary(unsigned long base, +static int msm8960_release_secondary(unsigned long base, unsigned int cpu) { void *base_ptr = ioremap_nocache(base + (cpu * 0x10000), SZ_4K); @@ -129,7 +129,7 @@ static int __cpuinit msm8960_release_secondary(unsigned long base, return 0; } -static int __cpuinit msm8974_release_secondary(unsigned long base, +static int msm8974_release_secondary(unsigned long base, unsigned int cpu) { void *base_ptr = ioremap_nocache(base + (cpu * 0x10000), SZ_4K); @@ -156,7 +156,7 @@ static int __cpuinit msm8974_release_secondary(unsigned long base, return 0; } -static int __cpuinit arm_release_secondary(unsigned long base, unsigned int cpu) +static int arm_release_secondary(unsigned long base, unsigned int cpu) { void *base_ptr = ioremap_nocache(base + (cpu * 0x10000), SZ_4K); if (!base_ptr) @@ -191,7 +191,7 @@ static int __cpuinit arm_release_secondary(unsigned long base, unsigned int cpu) return 0; } -static int __cpuinit release_from_pen(unsigned int cpu) +static int release_from_pen(unsigned int cpu) { unsigned long timeout; @@ -241,7 +241,7 @@ static int __cpuinit release_from_pen(unsigned int cpu) DEFINE_PER_CPU(int, cold_boot_done); -int __cpuinit scorpion_boot_secondary(unsigned int cpu, +int scorpion_boot_secondary(unsigned int cpu, struct task_struct *idle) { pr_debug("Starting secondary CPU %d\n", cpu); @@ -253,7 +253,7 @@ int __cpuinit scorpion_boot_secondary(unsigned int cpu, return release_from_pen(cpu); } -int __cpuinit msm8960_boot_secondary(unsigned int cpu, struct task_struct *idle) +int msm8960_boot_secondary(unsigned int cpu, struct task_struct *idle) { pr_debug("Starting secondary CPU %d\n", cpu); @@ -264,7 +264,7 @@ int __cpuinit msm8960_boot_secondary(unsigned int cpu, struct task_struct *idle) return release_from_pen(cpu); } -int __cpuinit msm8974_boot_secondary(unsigned int cpu, struct task_struct *idle) +int msm8974_boot_secondary(unsigned int cpu, struct task_struct *idle) { pr_debug("Starting secondary CPU %d\n", cpu); @@ -279,7 +279,7 @@ int __cpuinit msm8974_boot_secondary(unsigned int cpu, struct task_struct *idle) return release_from_pen(cpu); } -static int __cpuinit msm8916_boot_secondary(unsigned int cpu, +static int msm8916_boot_secondary(unsigned int cpu, struct task_struct *idle) { pr_debug("Starting secondary CPU %d\n", cpu); @@ -295,7 +295,7 @@ static int __cpuinit msm8916_boot_secondary(unsigned int cpu, return release_from_pen(cpu); } -static int __cpuinit msm8936_boot_secondary(unsigned int cpu, +static int msm8936_boot_secondary(unsigned int cpu, struct task_struct *idle) { int ret = 0; @@ -317,7 +317,7 @@ static int __cpuinit msm8936_boot_secondary(unsigned int cpu, return release_from_pen(cpu); } -int __cpuinit arm_boot_secondary(unsigned int cpu, struct task_struct *idle) +int arm_boot_secondary(unsigned int cpu, struct task_struct *idle) { pr_debug("Starting secondary CPU %d\n", cpu); diff --git a/arch/arm/mach-msm/timer.c b/arch/arm/mach-msm/timer.c index d749417c473d6..23d74cc68b952 100644 --- a/arch/arm/mach-msm/timer.c +++ b/arch/arm/mach-msm/timer.c @@ -889,7 +889,7 @@ static void __init msm_sched_clock_init(void) } #ifdef CONFIG_LOCAL_TIMERS -int __cpuinit local_timer_setup(struct clock_event_device *evt) +int local_timer_setup(struct clock_event_device *evt) { static DEFINE_PER_CPU(bool, first_boot) = true; struct msm_clock *clock = &msm_clocks[msm_global_timer]; diff --git a/arch/arm/mach-mvebu/platsmp.c b/arch/arm/mach-mvebu/platsmp.c index 875ea748391ca..703f9ccf435f5 100644 --- a/arch/arm/mach-mvebu/platsmp.c +++ b/arch/arm/mach-mvebu/platsmp.c @@ -71,12 +71,12 @@ void __init set_secondary_cpus_clock(void) } } -static void __cpuinit armada_xp_secondary_init(unsigned int cpu) +static void armada_xp_secondary_init(unsigned int cpu) { armada_xp_mpic_smp_cpu_init(); } -static int __cpuinit armada_xp_boot_secondary(unsigned int cpu, +static int armada_xp_boot_secondary(unsigned int cpu, struct task_struct *idle) { pr_info("Booting CPU %d\n", cpu); diff --git a/arch/arm/mach-omap2/omap-mpuss-lowpower.c b/arch/arm/mach-omap2/omap-mpuss-lowpower.c index e80327b6c81f6..1576ed36021d5 100644 --- a/arch/arm/mach-omap2/omap-mpuss-lowpower.c +++ b/arch/arm/mach-omap2/omap-mpuss-lowpower.c @@ -254,7 +254,7 @@ int omap4_enter_lowpower(unsigned int cpu, unsigned int power_state) * @cpu : CPU ID * @power_state: CPU low power state. */ -int __cpuinit omap4_hotplug_cpu(unsigned int cpu, unsigned int power_state) +int omap4_hotplug_cpu(unsigned int cpu, unsigned int power_state) { struct omap4_cpu_pm_info *pm_info = &per_cpu(omap4_pm_info, cpu); unsigned int cpu_state = 0; diff --git a/arch/arm/mach-omap2/omap-smp.c b/arch/arm/mach-omap2/omap-smp.c index 2a551f997aea5..818812bed623a 100644 --- a/arch/arm/mach-omap2/omap-smp.c +++ b/arch/arm/mach-omap2/omap-smp.c @@ -51,7 +51,7 @@ void __iomem *omap4_get_scu_base(void) return scu_base; } -static void __cpuinit omap4_secondary_init(unsigned int cpu) +static void omap4_secondary_init(unsigned int cpu) { /* * Configure ACTRL and enable NS SMP bit access on CPU1 on HS device. @@ -72,7 +72,7 @@ static void __cpuinit omap4_secondary_init(unsigned int cpu) spin_unlock(&boot_lock); } -static int __cpuinit omap4_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int omap4_boot_secondary(unsigned int cpu, struct task_struct *idle) { static struct clockdomain *cpu1_clkdm; static bool booted; diff --git a/arch/arm/mach-omap2/omap-wakeupgen.c b/arch/arm/mach-omap2/omap-wakeupgen.c index f8bb3b9b6a762..ed4b7d3bd5da5 100644 --- a/arch/arm/mach-omap2/omap-wakeupgen.c +++ b/arch/arm/mach-omap2/omap-wakeupgen.c @@ -323,7 +323,7 @@ static void irq_save_secure_context(void) #endif #ifdef CONFIG_HOTPLUG_CPU -static int __cpuinit irq_cpu_hotplug_notify(struct notifier_block *self, +static int irq_cpu_hotplug_notify(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned int)hcpu; diff --git a/arch/arm/mach-prima2/platsmp.c b/arch/arm/mach-prima2/platsmp.c index 1c3de7bed8413..3dbcb1ab6e37e 100644 --- a/arch/arm/mach-prima2/platsmp.c +++ b/arch/arm/mach-prima2/platsmp.c @@ -44,7 +44,7 @@ void __init sirfsoc_map_scu(void) scu_base = (void __iomem *)SIRFSOC_VA(base); } -static void __cpuinit sirfsoc_secondary_init(unsigned int cpu) +static void sirfsoc_secondary_init(unsigned int cpu) { /* * let the primary processor know we're out of the @@ -65,7 +65,7 @@ static struct of_device_id rsc_ids[] = { {}, }; -static int __cpuinit sirfsoc_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int sirfsoc_boot_secondary(unsigned int cpu, struct task_struct *idle) { unsigned long timeout; struct device_node *np; diff --git a/arch/arm/mach-shmobile/smp-emev2.c b/arch/arm/mach-shmobile/smp-emev2.c index e38691b4d0ddc..ee58c718752d1 100644 --- a/arch/arm/mach-shmobile/smp-emev2.c +++ b/arch/arm/mach-shmobile/smp-emev2.c @@ -30,7 +30,7 @@ #define EMEV2_SCU_BASE 0x1e000000 -static int __cpuinit emev2_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int emev2_boot_secondary(unsigned int cpu, struct task_struct *idle) { arch_send_wakeup_ipi_mask(cpumask_of(cpu_logical_map(cpu))); return 0; diff --git a/arch/arm/mach-shmobile/smp-r8a7779.c b/arch/arm/mach-shmobile/smp-r8a7779.c index a853bf182ed5e..77ee71ab2f1d2 100644 --- a/arch/arm/mach-shmobile/smp-r8a7779.c +++ b/arch/arm/mach-shmobile/smp-r8a7779.c @@ -81,7 +81,7 @@ static int r8a7779_platform_cpu_kill(unsigned int cpu) return ret ? ret : 1; } -static int __cpuinit r8a7779_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int r8a7779_boot_secondary(unsigned int cpu, struct task_struct *idle) { struct r8a7779_pm_ch *ch = NULL; int ret = -EIO; diff --git a/arch/arm/mach-shmobile/smp-sh73a0.c b/arch/arm/mach-shmobile/smp-sh73a0.c index 496592b6c7639..91584799e7056 100644 --- a/arch/arm/mach-shmobile/smp-sh73a0.c +++ b/arch/arm/mach-shmobile/smp-sh73a0.c @@ -48,7 +48,7 @@ void __init sh73a0_register_twd(void) } #endif -static int __cpuinit sh73a0_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int sh73a0_boot_secondary(unsigned int cpu, struct task_struct *idle) { cpu = cpu_logical_map(cpu); diff --git a/arch/arm/mach-socfpga/platsmp.c b/arch/arm/mach-socfpga/platsmp.c index b51ce8c7929dc..5356a72bc8ce9 100644 --- a/arch/arm/mach-socfpga/platsmp.c +++ b/arch/arm/mach-socfpga/platsmp.c @@ -29,7 +29,7 @@ #include "core.h" -static int __cpuinit socfpga_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int socfpga_boot_secondary(unsigned int cpu, struct task_struct *idle) { int trampoline_size = &secondary_trampoline_end - &secondary_trampoline; diff --git a/arch/arm/mach-spear/generic.h b/arch/arm/mach-spear/generic.h index 904f2c907b46d..a99d90a4d09c5 100644 --- a/arch/arm/mach-spear/generic.h +++ b/arch/arm/mach-spear/generic.h @@ -37,7 +37,7 @@ void __init spear13xx_l2x0_init(void); void spear_restart(enum reboot_mode, const char *); void spear13xx_secondary_startup(void); -void __cpuinit spear13xx_cpu_die(unsigned int cpu); +void spear13xx_cpu_die(unsigned int cpu); extern struct smp_operations spear13xx_smp_ops; diff --git a/arch/arm/mach-spear/platsmp.c b/arch/arm/mach-spear/platsmp.c index 9c4c722c954e0..5c4a19887b2bb 100644 --- a/arch/arm/mach-spear/platsmp.c +++ b/arch/arm/mach-spear/platsmp.c @@ -24,7 +24,7 @@ static DEFINE_SPINLOCK(boot_lock); static void __iomem *scu_base = IOMEM(VA_SCU_BASE); -static void __cpuinit spear13xx_secondary_init(unsigned int cpu) +static void spear13xx_secondary_init(unsigned int cpu) { /* * let the primary processor know we're out of the @@ -40,7 +40,7 @@ static void __cpuinit spear13xx_secondary_init(unsigned int cpu) spin_unlock(&boot_lock); } -static int __cpuinit spear13xx_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int spear13xx_boot_secondary(unsigned int cpu, struct task_struct *idle) { unsigned long timeout; diff --git a/arch/arm/mach-tegra/platsmp.c b/arch/arm/mach-tegra/platsmp.c index fad4226ef710d..8aa84b50a4bba 100644 --- a/arch/arm/mach-tegra/platsmp.c +++ b/arch/arm/mach-tegra/platsmp.c @@ -35,7 +35,7 @@ static cpumask_t tegra_cpu_init_mask; -static void __cpuinit tegra_secondary_init(unsigned int cpu) +static void tegra_secondary_init(unsigned int cpu) { cpumask_set_cpu(cpu, &tegra_cpu_init_mask); } @@ -144,7 +144,7 @@ static int tegra114_boot_secondary(unsigned int cpu, struct task_struct *idle) return tegra_pmc_cpu_power_on(cpu); } -static int __cpuinit tegra_boot_secondary(unsigned int cpu, +static int tegra_boot_secondary(unsigned int cpu, struct task_struct *idle) { if (IS_ENABLED(CONFIG_ARCH_TEGRA_2x_SOC) && tegra_chip_id == TEGRA20) diff --git a/arch/arm/mach-tegra/pm.c b/arch/arm/mach-tegra/pm.c index e7594fa85066a..9718125a68a5b 100644 --- a/arch/arm/mach-tegra/pm.c +++ b/arch/arm/mach-tegra/pm.c @@ -175,7 +175,7 @@ static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = { [TEGRA_SUSPEND_LP0] = "LP0", }; -static int __cpuinit tegra_suspend_enter(suspend_state_t state) +static int tegra_suspend_enter(suspend_state_t state) { enum tegra_suspend_mode mode = tegra_pmc_get_suspend_mode(); diff --git a/arch/arm/mach-ux500/platsmp.c b/arch/arm/mach-ux500/platsmp.c index 14d90469392f1..1f296e796a4fe 100644 --- a/arch/arm/mach-ux500/platsmp.c +++ b/arch/arm/mach-ux500/platsmp.c @@ -54,7 +54,7 @@ static void __iomem *scu_base_addr(void) static DEFINE_SPINLOCK(boot_lock); -static void __cpuinit ux500_secondary_init(unsigned int cpu) +static void ux500_secondary_init(unsigned int cpu) { /* * let the primary processor know we're out of the @@ -69,7 +69,7 @@ static void __cpuinit ux500_secondary_init(unsigned int cpu) spin_unlock(&boot_lock); } -static int __cpuinit ux500_boot_secondary(unsigned int cpu, struct task_struct *idle) +static int ux500_boot_secondary(unsigned int cpu, struct task_struct *idle) { unsigned long timeout; diff --git a/arch/arm/mach-zynq/common.h b/arch/arm/mach-zynq/common.h index fbbd0e21c404d..3040d219570f2 100644 --- a/arch/arm/mach-zynq/common.h +++ b/arch/arm/mach-zynq/common.h @@ -27,7 +27,7 @@ extern void secondary_startup(void); extern char zynq_secondary_trampoline; extern char zynq_secondary_trampoline_jump; extern char zynq_secondary_trampoline_end; -extern int __cpuinit zynq_cpun_start(u32 address, int cpu); +extern int zynq_cpun_start(u32 address, int cpu); extern struct smp_operations zynq_smp_ops __initdata; #endif diff --git a/arch/arm/mach-zynq/platsmp.c b/arch/arm/mach-zynq/platsmp.c index 5fc167e076199..6bc2270b35704 100644 --- a/arch/arm/mach-zynq/platsmp.c +++ b/arch/arm/mach-zynq/platsmp.c @@ -30,11 +30,11 @@ /* * Store number of cores in the system * Because of scu_get_core_count() must be in __init section and can't - * be called from zynq_cpun_start() because it is in __cpuinit section. + * be called from zynq_cpun_start() because it is in section. */ static int ncores; -int __cpuinit zynq_cpun_start(u32 address, int cpu) +int zynq_cpun_start(u32 address, int cpu) { u32 trampoline_code_size = &zynq_secondary_trampoline_end - &zynq_secondary_trampoline; @@ -92,7 +92,7 @@ int __cpuinit zynq_cpun_start(u32 address, int cpu) } EXPORT_SYMBOL(zynq_cpun_start); -static int __cpuinit zynq_boot_secondary(unsigned int cpu, +static int zynq_boot_secondary(unsigned int cpu, struct task_struct *idle) { return zynq_cpun_start(virt_to_phys(secondary_startup), cpu); diff --git a/arch/arm/plat-versatile/platsmp.c b/arch/arm/plat-versatile/platsmp.c index 1e1b2d7697488..39895d892c3be 100644 --- a/arch/arm/plat-versatile/platsmp.c +++ b/arch/arm/plat-versatile/platsmp.c @@ -23,7 +23,7 @@ * observers, irrespective of whether they're taking part in coherency * or not. This is necessary for the hotplug code to work reliably. */ -static void __cpuinit write_pen_release(int val) +static void write_pen_release(int val) { pen_release = val; smp_wmb(); @@ -33,7 +33,7 @@ static void __cpuinit write_pen_release(int val) static DEFINE_SPINLOCK(boot_lock); -void __cpuinit versatile_secondary_init(unsigned int cpu) +void versatile_secondary_init(unsigned int cpu) { /* * let the primary processor know we're out of the @@ -48,7 +48,7 @@ void __cpuinit versatile_secondary_init(unsigned int cpu) spin_unlock(&boot_lock); } -int __cpuinit versatile_boot_secondary(unsigned int cpu, struct task_struct *idle) +int versatile_boot_secondary(unsigned int cpu, struct task_struct *idle) { unsigned long timeout; diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 61cc7178e2b28..1d0f32bb4f1b6 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -407,12 +407,6 @@ config SCHED_SMT MultiThreading at a cost of slightly increased overhead in some places. If unsure say N here. -config ARCH_WANTS_CTXSW_LOGGING - bool "Enable logging in context switch" - depends on SMP - help - Logs critical context switch latencies. If unsure say N here. - config NR_CPUS int "Maximum number of CPUs (2-32)" range 2 32 diff --git a/arch/arm64/configs/oneplus2_defconfig b/arch/arm64/configs/oneplus2_defconfig index 6ad8080730f8a..b33447f962596 100644 --- a/arch/arm64/configs/oneplus2_defconfig +++ b/arch/arm64/configs/oneplus2_defconfig @@ -1,5 +1,9 @@ CONFIG_LOCALVERSION="-perf" -CONFIG_SYSVIPC=y +CONFIG_SYSVIPC=n +CONFIG_SCHED_TUNE=y +CONFIG_CGROUP_SCHEDTUNE=y +CONFIG_CPU_FREQ_GOV_SCHED=y +CONFIG_RT_MUTEXES=y CONFIG_AUDIT=y CONFIG_NO_HZ=y CONFIG_HIGH_RES_TIMERS=y @@ -12,9 +16,9 @@ CONFIG_CPUSETS=y CONFIG_CGROUP_CPUACCT=y CONFIG_RESOURCE_COUNTERS=y CONFIG_CGROUP_SCHED=y -CONFIG_CFS_BANDWIDTH=y +CONFIG_CFS_BANDWIDTH=n CONFIG_RT_GROUP_SCHED=y -CONFIG_SCHED_HMP=y +CONFIG_SCHED_HMP=n CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_IPC_NS is not set @@ -503,7 +507,7 @@ CONFIG_MSM_SYSTEM_HEALTH_MONITOR=y CONFIG_QCOM_EARLY_RANDOM=y CONFIG_MSM_CORE_CTL_HELPER=y CONFIG_MSM_PERFORMANCE=y -CONFIG_ONEPLUS_HCUBE=m +CONFIG_ONEPLUS_HCUBE=n CONFIG_QCOM_NPA_DUMP=y CONFIG_MSM_TZ_LOG=y CONFIG_PARAM_READ_WRITE=y diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h new file mode 100644 index 0000000000000..5c89df0acbcb2 --- /dev/null +++ b/arch/arm64/include/asm/sysreg.h @@ -0,0 +1,60 @@ +/* + * Macros for accessing system registers with older binutils. + * + * Copyright (C) 2014 ARM Ltd. + * Author: Catalin Marinas + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __ASM_SYSREG_H +#define __ASM_SYSREG_H + +#define sys_reg(op0, op1, crn, crm, op2) \ + ((((op0)-2)<<19)|((op1)<<16)|((crn)<<12)|((crm)<<8)|((op2)<<5)) + +#ifdef __ASSEMBLY__ + + .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 + .equ __reg_num_x\num, \num + .endr + .equ __reg_num_xzr, 31 + + .macro mrs_s, rt, sreg + .inst 0xd5300000|(\sreg)|(__reg_num_\rt) + .endm + + .macro msr_s, sreg, rt + .inst 0xd5100000|(\sreg)|(__reg_num_\rt) + .endm + +#else + +asm( +" .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30\n" +" .equ __reg_num_x\\num, \\num\n" +" .endr\n" +" .equ __reg_num_xzr, 31\n" +"\n" +" .macro mrs_s, rt, sreg\n" +" .inst 0xd5300000|(\\sreg)|(__reg_num_\\rt)\n" +" .endm\n" +"\n" +" .macro msr_s, sreg, rt\n" +" .inst 0xd5100000|(\\sreg)|(__reg_num_\\rt)\n" +" .endm\n" +); + +#endif + +#endif /* __ASM_SYSREG_H */ diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index 7ebcd31ce51ca..4ea11bad40025 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -24,6 +24,15 @@ void init_cpu_topology(void); void store_cpu_topology(unsigned int cpuid); const struct cpumask *cpu_coregroup_mask(int cpu); +struct sched_domain; +#ifdef CONFIG_CPU_FREQ +#define arch_scale_freq_capacity cpufreq_scale_freq_capacity +extern unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu); +extern unsigned long cpufreq_scale_max_freq_capacity(int cpu); +#endif +#define arch_scale_cpu_capacity scale_cpu_capacity +extern unsigned long scale_cpu_capacity(struct sched_domain *sd, int cpu); + #else static inline void init_cpu_topology(void) { } diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index bb1878871b9a8..2eaec55fcfbf2 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -139,7 +139,7 @@ static void clear_os_lock(void *unused) asm volatile("msr oslar_el1, %0" : : "r" (0)); } -static int __cpuinit os_lock_notify(struct notifier_block *self, +static int os_lock_notify(struct notifier_block *self, unsigned long action, void *data) { int cpu = (unsigned long)data; @@ -148,11 +148,11 @@ static int __cpuinit os_lock_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata os_lock_nb = { +static struct notifier_block os_lock_nb = { .notifier_call = os_lock_notify, }; -static int __cpuinit debug_monitors_init(void) +static int debug_monitors_init(void) { cpu_notifier_register_begin(); diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c index de7b854438d79..db3b6fd716bc6 100644 --- a/arch/arm64/kernel/hw_breakpoint.c +++ b/arch/arm64/kernel/hw_breakpoint.c @@ -880,7 +880,7 @@ static void hw_breakpoint_reset(void *unused) } } -static int __cpuinit hw_breakpoint_reset_notify(struct notifier_block *self, +static int hw_breakpoint_reset_notify(struct notifier_block *self, unsigned long action, void *hcpu) { @@ -890,7 +890,7 @@ static int __cpuinit hw_breakpoint_reset_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata hw_breakpoint_reset_nb = { +static struct notifier_block hw_breakpoint_reset_nb = { .notifier_call = hw_breakpoint_reset_notify, }; diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c index 21f3b11136bed..fc77c1e4c656e 100644 --- a/arch/arm64/kernel/perf_event.c +++ b/arch/arm64/kernel/perf_event.c @@ -1519,8 +1519,8 @@ static void armpmu_hotplug_disable(void *parm_pmu) * UNKNOWN at reset, the PMU must be explicitly reset to avoid reading * junk values out of them. */ -static int __cpuinit cpu_pmu_notify(struct notifier_block *b, - unsigned long action, void *hcpu) +static int cpu_pmu_notify(struct notifier_block *b, + unsigned long action, void *hcpu) { int irq; struct pmu *pmu; diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 73ad727ccd566..9c7c1545fd494 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -74,7 +74,7 @@ enum ipi_msg_type { * Boot a secondary CPU, and assign it the specified idle task. * This also gives us the initial stack to use for this CPU. */ -static int __cpuinit boot_secondary(unsigned int cpu, struct task_struct *idle) +static int boot_secondary(unsigned int cpu, struct task_struct *idle) { if (cpu_ops[cpu]->cpu_boot) return cpu_ops[cpu]->cpu_boot(cpu); @@ -84,7 +84,7 @@ static int __cpuinit boot_secondary(unsigned int cpu, struct task_struct *idle) static DECLARE_COMPLETION(cpu_running); -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) +int __cpu_up(unsigned int cpu, struct task_struct *idle) { int ret; @@ -129,7 +129,7 @@ static void smp_store_cpu_info(unsigned int cpuid) * This is the secondary CPU boot entry. We're using this CPUs * idle thread stack, but a set of temporary page tables. */ -asmlinkage void __cpuinit secondary_start_kernel(void) +asmlinkage void secondary_start_kernel(void) { struct mm_struct *mm = &init_mm; unsigned int cpu = smp_processor_id(); diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 75e586914d8b2..dbee05e376626 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -20,32 +20,22 @@ #include #include #include +#include #include #include #include -/* - * cpu power table - * This per cpu data structure describes the relative capacity of each core. - * On a heteregenous system, cores don't have the same computation capacity - * and we reflect that difference in the cpu_power field so the scheduler can - * take this difference into account during load balance. A per cpu structure - * is preferred because each CPU updates its own cpu_power field during the - * load balance except for idle cores. One idle core is selected to run the - * rebalance_domains for all idle cores and the cpu_power can be updated - * during this sequence. - */ -static DEFINE_PER_CPU(unsigned long, cpu_scale); +static DEFINE_PER_CPU(unsigned long, cpu_scale) = SCHED_CAPACITY_SCALE; -unsigned long arch_scale_freq_power(struct sched_domain *sd, int cpu) +unsigned long scale_cpu_capacity(struct sched_domain *sd, int cpu) { return per_cpu(cpu_scale, cpu); } -static void set_power_scale(unsigned int cpu, unsigned long power) +static void set_capacity_scale(unsigned int cpu, unsigned long capacity) { - per_cpu(cpu_scale, cpu) = power; + per_cpu(cpu_scale, cpu) = capacity; } static int __init get_cpu_for_node(struct device_node *node) @@ -186,47 +176,6 @@ static int __init parse_cluster(struct device_node *cluster, int depth) return 0; } -struct cpu_efficiency { - const char *compatible; - unsigned long efficiency; -}; - -/* - * Table of relative efficiency of each processors - * The efficiency value must fit in 20bit and the final - * cpu_scale value must be in the range - * 0 < cpu_scale < 3*SCHED_POWER_SCALE/2 - * in order to return at most 1 when DIV_ROUND_CLOSEST - * is used to compute the capacity of a CPU. - * Processors that are not defined in the table, - * use the default SCHED_POWER_SCALE value for cpu_scale. - */ -static const struct cpu_efficiency table_efficiency[] = { - { "arm,cortex-a57", 3891 }, - { "arm,cortex-a53", 2048 }, - { NULL, }, -}; - -static unsigned long *__cpu_capacity; -#define cpu_capacity(cpu) __cpu_capacity[cpu] - -static unsigned long middle_capacity = 1; - -static DEFINE_PER_CPU(unsigned long, cpu_efficiency) = SCHED_POWER_SCALE; - -unsigned long arch_get_cpu_efficiency(int cpu) -{ - return per_cpu(cpu_efficiency, cpu); -} - -/* - * Iterate all CPUs' descriptor in DT and compute the efficiency - * (as per table_efficiency). Also calculate a middle efficiency - * as close as possible to (max{eff_i} - min{eff_i}) / 2 - * This is later used to scale the cpu_power field such that an - * 'average' CPU is of middle power. Also see the comments near - * table_efficiency[] and update_cpu_power(). - */ static int __init parse_dt_topology(void) { struct device_node *cn, *map; @@ -270,102 +219,42 @@ static int __init parse_dt_topology(void) return ret; } -static void __init parse_dt_cpu_power(void) -{ - const struct cpu_efficiency *cpu_eff; - struct device_node *cn; - unsigned long min_capacity = ULONG_MAX; - unsigned long max_capacity = 0; - unsigned long capacity = 0; - int cpu; - - __cpu_capacity = kcalloc(nr_cpu_ids, sizeof(*__cpu_capacity), - GFP_NOWAIT); - - for_each_possible_cpu(cpu) { - const u32 *rate; - int len; - - /* Too early to use cpu->of_node */ - cn = of_get_cpu_node(cpu, NULL); - if (!cn) { - pr_err("Missing device node for CPU %d\n", cpu); - continue; - } - - for (cpu_eff = table_efficiency; cpu_eff->compatible; cpu_eff++) - if (of_device_is_compatible(cn, cpu_eff->compatible)) - break; - - if (cpu_eff->compatible == NULL) { - pr_warn("%s: Unknown CPU type\n", cn->full_name); - continue; - } - - per_cpu(cpu_efficiency, cpu) = cpu_eff->efficiency; - - rate = of_get_property(cn, "clock-frequency", &len); - if (!rate || len != 4) { - pr_err("%s: Missing clock-frequency property\n", - cn->full_name); - continue; - } - - capacity = ((be32_to_cpup(rate)) >> 20) * cpu_eff->efficiency; - - /* Save min capacity of the system */ - if (capacity < min_capacity) - min_capacity = capacity; +/* + * cpu topology table + */ +struct cpu_topology cpu_topology[NR_CPUS]; +EXPORT_SYMBOL_GPL(cpu_topology); - /* Save max capacity of the system */ - if (capacity > max_capacity) - max_capacity = capacity; +static inline const struct sched_group_energy *cpu_core_energy(int cpu) +{ + struct sched_group_energy *sge = sge_array[cpu][SD_LEVEL0]; - cpu_capacity(cpu) = capacity; + if (!sge) { + pr_warn("Invalid sched_group_energy for CPU%d\n", cpu); + return NULL; } - /* If min and max capacities are equal we bypass the update of the - * cpu_scale because all CPUs have the same capacity. Otherwise, we - * compute a middle_capacity factor that will ensure that the capacity - * of an 'average' CPU of the system will be as close as possible to - * SCHED_POWER_SCALE, which is the default value, but with the - * constraint explained near table_efficiency[]. - */ - if (min_capacity == max_capacity) - return; - else if (4 * max_capacity < (3 * (max_capacity + min_capacity))) - middle_capacity = (min_capacity + max_capacity) - >> (SCHED_POWER_SHIFT+1); - else - middle_capacity = ((max_capacity / 3) - >> (SCHED_POWER_SHIFT-1)) + 1; + return sge; } -/* - * Look for a customed capacity of a CPU in the cpu_topo_data table during the - * boot. The update of all CPUs is in O(n^2) for heteregeneous system but the - * function returns directly for SMP system. - */ -static void update_cpu_power(unsigned int cpu) +const struct cpumask *cpu_coregroup_mask(int cpu) { - if (!cpu_capacity(cpu)) - return; + return &cpu_topology[cpu].core_sibling; +} - set_power_scale(cpu, cpu_capacity(cpu) / middle_capacity); +static void update_cpu_capacity(unsigned int cpu) +{ + unsigned long capacity = SCHED_CAPACITY_SCALE; - pr_info("CPU%u: update cpu_power %lu\n", - cpu, arch_scale_freq_power(NULL, cpu)); -} + if (cpu_core_energy(cpu)) { + int max_cap_idx = cpu_core_energy(cpu)->nr_cap_states - 1; + capacity = cpu_core_energy(cpu)->cap_states[max_cap_idx].cap; + } -/* - * cpu topology table - */ -struct cpu_topology cpu_topology[NR_CPUS]; -EXPORT_SYMBOL_GPL(cpu_topology); + set_capacity_scale(cpu, capacity); -const struct cpumask *cpu_coregroup_mask(int cpu) -{ - return &cpu_topology[cpu].core_sibling; + pr_info("CPU%d: update cpu_capacity %lu\n", + cpu, arch_scale_cpu_capacity(NULL, cpu)); } static void update_siblings_masks(unsigned int cpuid) @@ -440,7 +329,7 @@ void store_cpu_topology(unsigned int cpuid) topology_populated: update_siblings_masks(cpuid); - update_cpu_power(cpuid); + update_cpu_capacity(cpuid); } static void __init reset_cpu_topology(void) @@ -461,14 +350,32 @@ static void __init reset_cpu_topology(void) } } -static void __init reset_cpu_power(void) +static inline const struct sched_group_energy *cpu_cluster_energy(int cpu) { - unsigned int cpu; + struct sched_group_energy *sge = sge_array[cpu][SD_LEVEL1]; - for_each_possible_cpu(cpu) - set_power_scale(cpu, SCHED_POWER_SCALE); + if (!sge) { + pr_warn("Invalid sched_group_energy for Cluster%d\n", cpu); + return NULL; + } + + return sge; +} + +static inline int cpu_corepower_flags(void) +{ + return SD_SHARE_PKG_RESOURCES | SD_SHARE_POWERDOMAIN | \ + SD_SHARE_CAP_STATES; } +static struct sched_domain_topology_level arm64_topology[] = { +#ifdef CONFIG_SCHED_MC + { cpu_coregroup_mask, cpu_corepower_flags, cpu_core_energy, SD_INIT_NAME(MC) }, +#endif + { cpu_cpu_mask, 0, cpu_cluster_energy, SD_INIT_NAME(DIE) }, + { NULL, }, +}; + void __init init_cpu_topology(void) { reset_cpu_topology(); @@ -479,7 +386,8 @@ void __init init_cpu_topology(void) */ if (parse_dt_topology()) reset_cpu_topology(); + else + set_sched_topology(arm64_topology); - reset_cpu_power(); - parse_dt_cpu_power(); + init_sched_energy_costs(); } diff --git a/arch/blackfin/kernel/perf_event.c b/arch/blackfin/kernel/perf_event.c index e47d19ae3e063..974e55496db30 100644 --- a/arch/blackfin/kernel/perf_event.c +++ b/arch/blackfin/kernel/perf_event.c @@ -468,7 +468,7 @@ static void bfin_pmu_setup(int cpu) memset(cpuhw, 0, sizeof(struct cpu_hw_events)); } -static int __cpuinit +static int bfin_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (long)hcpu; diff --git a/arch/blackfin/kernel/setup.c b/arch/blackfin/kernel/setup.c index 107b306b06f10..19ad0637e8ff0 100644 --- a/arch/blackfin/kernel/setup.c +++ b/arch/blackfin/kernel/setup.c @@ -99,7 +99,7 @@ void __init generate_cplb_tables(void) } #endif -void __cpuinit bfin_setup_caches(unsigned int cpu) +void bfin_setup_caches(unsigned int cpu) { #ifdef CONFIG_BFIN_ICACHE bfin_icache_init(icplb_tbl[cpu]); @@ -165,7 +165,7 @@ void __cpuinit bfin_setup_caches(unsigned int cpu) #endif } -void __cpuinit bfin_setup_cpudata(unsigned int cpu) +void bfin_setup_cpudata(unsigned int cpu) { struct blackfin_cpudata *cpudata = &per_cpu(cpu_data, cpu); diff --git a/arch/blackfin/mach-bf561/smp.c b/arch/blackfin/mach-bf561/smp.c index ab1c617b9cfc9..7748b84f1e541 100644 --- a/arch/blackfin/mach-bf561/smp.c +++ b/arch/blackfin/mach-bf561/smp.c @@ -48,7 +48,7 @@ int __init setup_profiling_timer(unsigned int multiplier) /* not supported */ return -EINVAL; } -void __cpuinit platform_secondary_init(unsigned int cpu) +void platform_secondary_init(unsigned int cpu) { /* Clone setup for peripheral interrupt sources from CoreA. */ bfin_write_SICB_IMASK0(bfin_read_SIC_IMASK0()); @@ -74,7 +74,7 @@ void __cpuinit platform_secondary_init(unsigned int cpu) spin_unlock(&boot_lock); } -int __cpuinit platform_boot_secondary(unsigned int cpu, struct task_struct *idle) +int platform_boot_secondary(unsigned int cpu, struct task_struct *idle) { unsigned long timeout; @@ -155,7 +155,7 @@ void platform_clear_ipi(unsigned int cpu, int irq) * Setup core B's local core timer. * In SMP, core timer is used for clock event device. */ -void __cpuinit bfin_local_timer_setup(void) +void bfin_local_timer_setup(void) { #if defined(CONFIG_TICKSOURCE_CORETMR) struct irq_data *data = irq_get_irq_data(IRQ_CORETMR); diff --git a/arch/blackfin/mach-common/cache-c.c b/arch/blackfin/mach-common/cache-c.c index a60a24f5035d3..0e1e451fd7d81 100644 --- a/arch/blackfin/mach-common/cache-c.c +++ b/arch/blackfin/mach-common/cache-c.c @@ -52,7 +52,7 @@ bfin_cache_init(struct cplb_entry *cplb_tbl, unsigned long cplb_addr, } #ifdef CONFIG_BFIN_ICACHE -void __cpuinit bfin_icache_init(struct cplb_entry *icplb_tbl) +void bfin_icache_init(struct cplb_entry *icplb_tbl) { bfin_cache_init(icplb_tbl, ICPLB_ADDR0, ICPLB_DATA0, IMEM_CONTROL, (IMC | ENICPLB)); @@ -60,7 +60,7 @@ void __cpuinit bfin_icache_init(struct cplb_entry *icplb_tbl) #endif #ifdef CONFIG_BFIN_DCACHE -void __cpuinit bfin_dcache_init(struct cplb_entry *dcplb_tbl) +void bfin_dcache_init(struct cplb_entry *dcplb_tbl) { /* * Anomaly notes: diff --git a/arch/blackfin/mach-common/ints-priority.c b/arch/blackfin/mach-common/ints-priority.c index 6c0c6816a51a1..d143fd8d2bc50 100644 --- a/arch/blackfin/mach-common/ints-priority.c +++ b/arch/blackfin/mach-common/ints-priority.c @@ -1281,7 +1281,7 @@ static struct irq_chip bfin_gpio_irqchip = { .irq_set_wake = bfin_gpio_set_wake, }; -void __cpuinit init_exception_vectors(void) +void init_exception_vectors(void) { /* cannot program in software: * evt0 - emulation (jtag) diff --git a/arch/blackfin/mach-common/smp.c b/arch/blackfin/mach-common/smp.c index 1bc2ce6f3c947..55d4cfb40faaa 100644 --- a/arch/blackfin/mach-common/smp.c +++ b/arch/blackfin/mach-common/smp.c @@ -46,7 +46,7 @@ struct corelock_slot corelock __attribute__ ((__section__(".l2.bss"))); unsigned long blackfin_iflush_l1_entry[NR_CPUS]; #endif -struct blackfin_initial_pda __cpuinitdata initial_pda_coreb; +struct blackfin_initial_pda initial_pda_coreb; enum ipi_message_type { BFIN_IPI_TIMER, @@ -249,7 +249,7 @@ void smp_send_stop(void) return; } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) +int __cpu_up(unsigned int cpu, struct task_struct *idle) { int ret; @@ -262,7 +262,7 @@ int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) return ret; } -static void __cpuinit setup_secondary(unsigned int cpu) +static void setup_secondary(unsigned int cpu) { unsigned long ilat; @@ -280,7 +280,7 @@ static void __cpuinit setup_secondary(unsigned int cpu) IMASK_IVG10 | IMASK_IVG9 | IMASK_IVG8 | IMASK_IVG7 | IMASK_IVGHW; } -void __cpuinit secondary_start_kernel(void) +void secondary_start_kernel(void) { unsigned int cpu = smp_processor_id(); struct mm_struct *mm = &init_mm; @@ -404,7 +404,7 @@ EXPORT_SYMBOL(resync_core_dcache); #endif #ifdef CONFIG_HOTPLUG_CPU -int __cpuexit __cpu_disable(void) +int __cpu_disable(void) { unsigned int cpu = smp_processor_id(); @@ -417,7 +417,7 @@ int __cpuexit __cpu_disable(void) static DECLARE_COMPLETION(cpu_killed); -int __cpuexit __cpu_die(unsigned int cpu) +int __cpu_die(unsigned int cpu) { return wait_for_completion_timeout(&cpu_killed, 5000); } diff --git a/arch/cris/arch-v32/kernel/smp.c b/arch/cris/arch-v32/kernel/smp.c index cdd12028de0c5..fe8e6039db2a0 100644 --- a/arch/cris/arch-v32/kernel/smp.c +++ b/arch/cris/arch-v32/kernel/smp.c @@ -197,7 +197,7 @@ int setup_profiling_timer(unsigned int multiplier) */ unsigned long cache_decay_ticks = 1; -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { smp_boot_one_cpu(cpu, tidle); return cpu_online(cpu) ? 0 : -ENOSYS; diff --git a/arch/frv/kernel/setup.c b/arch/frv/kernel/setup.c index a5136474c6fd6..f908aab9f3b91 100644 --- a/arch/frv/kernel/setup.c +++ b/arch/frv/kernel/setup.c @@ -709,7 +709,7 @@ static void __init reserve_dma_coherent(void) /* * calibrate the delay loop */ -void __cpuinit calibrate_delay(void) +void calibrate_delay(void) { loops_per_jiffy = __delay_loops_MHz * (1000000 / HZ); diff --git a/arch/hexagon/kernel/setup.c b/arch/hexagon/kernel/setup.c index bfe13311d70d5..29d1f1b000165 100644 --- a/arch/hexagon/kernel/setup.c +++ b/arch/hexagon/kernel/setup.c @@ -41,7 +41,7 @@ static char default_command_line[COMMAND_LINE_SIZE] __initdata = CONFIG_CMDLINE; int on_simulator; -void __cpuinit calibrate_delay(void) +void calibrate_delay(void) { loops_per_jiffy = thread_freq_mhz * 1000000 / HZ; } diff --git a/arch/hexagon/kernel/smp.c b/arch/hexagon/kernel/smp.c index 0e364ca431981..9faaa940452b9 100644 --- a/arch/hexagon/kernel/smp.c +++ b/arch/hexagon/kernel/smp.c @@ -146,7 +146,7 @@ void __init smp_prepare_boot_cpu(void) * to point to current thread info */ -void __cpuinit start_secondary(void) +void start_secondary(void) { unsigned int cpu; unsigned long thread_ptr; @@ -194,7 +194,7 @@ void __cpuinit start_secondary(void) * maintains control until "cpu_online(cpu)" is set. */ -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) +int __cpu_up(unsigned int cpu, struct task_struct *idle) { struct thread_info *thread = (struct thread_info *)idle->stack; void *stack_start; diff --git a/arch/ia64/kernel/acpi.c b/arch/ia64/kernel/acpi.c index 335eb07480fe9..5eb71d22c3d59 100644 --- a/arch/ia64/kernel/acpi.c +++ b/arch/ia64/kernel/acpi.c @@ -807,7 +807,7 @@ int acpi_isa_irq_to_gsi(unsigned isa_irq, u32 *gsi) * ACPI based hotplug CPU support */ #ifdef CONFIG_ACPI_HOTPLUG_CPU -static __cpuinit +static int acpi_map_cpu2node(acpi_handle handle, int cpu, int physid) { #ifdef CONFIG_ACPI_NUMA @@ -882,7 +882,7 @@ __init void prefill_possible_map(void) set_cpu_possible(i, true); } -static int __cpuinit _acpi_map_lsapic(acpi_handle handle, int *pcpu) +static int _acpi_map_lsapic(acpi_handle handle, int *pcpu) { struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; union acpi_object *obj; diff --git a/arch/ia64/kernel/err_inject.c b/arch/ia64/kernel/err_inject.c index 2d67317a1ec20..4dc334c71dbd6 100644 --- a/arch/ia64/kernel/err_inject.c +++ b/arch/ia64/kernel/err_inject.c @@ -225,17 +225,17 @@ static struct attribute_group err_inject_attr_group = { .name = "err_inject" }; /* Add/Remove err_inject interface for CPU device */ -static int __cpuinit err_inject_add_dev(struct device * sys_dev) +static int err_inject_add_dev(struct device * sys_dev) { return sysfs_create_group(&sys_dev->kobj, &err_inject_attr_group); } -static int __cpuinit err_inject_remove_dev(struct device * sys_dev) +static int err_inject_remove_dev(struct device * sys_dev) { sysfs_remove_group(&sys_dev->kobj, &err_inject_attr_group); return 0; } -static int __cpuinit err_inject_cpu_callback(struct notifier_block *nfb, +static int err_inject_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; @@ -256,7 +256,7 @@ static int __cpuinit err_inject_cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata err_inject_cpu_notifier = +static struct notifier_block err_inject_cpu_notifier = { .notifier_call = err_inject_cpu_callback, }; diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c index d7396dbb07bb4..b8edfa75a83f9 100644 --- a/arch/ia64/kernel/mca.c +++ b/arch/ia64/kernel/mca.c @@ -631,7 +631,7 @@ ia64_mca_register_cpev (int cpev) * Outputs * None */ -void __cpuinit +void ia64_mca_cmc_vector_setup (void) { cmcv_reg_t cmcv; @@ -1814,7 +1814,7 @@ static struct irqaction mca_cpep_irqaction = { * format most of the fields. */ -static void __cpuinit +static void format_mca_init_stack(void *mca_data, unsigned long offset, const char *type, int cpu) { @@ -1844,7 +1844,7 @@ static void * __init_refok mca_bootmem(void) } /* Do per-CPU MCA-related initialization. */ -void __cpuinit +void ia64_mca_cpu_init(void *cpu_data) { void *pal_vaddr; @@ -1896,7 +1896,7 @@ ia64_mca_cpu_init(void *cpu_data) PAGE_KERNEL)); } -static void __cpuinit ia64_mca_cmc_vector_adjust(void *dummy) +static void ia64_mca_cmc_vector_adjust(void *dummy) { unsigned long flags; @@ -1906,7 +1906,7 @@ static void __cpuinit ia64_mca_cmc_vector_adjust(void *dummy) local_irq_restore(flags); } -static int __cpuinit mca_cpu_callback(struct notifier_block *nfb, +static int mca_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { @@ -1922,7 +1922,7 @@ static int __cpuinit mca_cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block mca_cpu_notifier __cpuinitdata = { +static struct notifier_block mca_cpu_notifier = { .notifier_call = mca_cpu_callback }; diff --git a/arch/ia64/kernel/numa.c b/arch/ia64/kernel/numa.c index c93420c97409d..d288cde936066 100644 --- a/arch/ia64/kernel/numa.c +++ b/arch/ia64/kernel/numa.c @@ -30,7 +30,7 @@ EXPORT_SYMBOL(cpu_to_node_map); cpumask_t node_to_cpu_mask[MAX_NUMNODES] __cacheline_aligned; EXPORT_SYMBOL(node_to_cpu_mask); -void __cpuinit map_cpu_to_node(int cpu, int nid) +void map_cpu_to_node(int cpu, int nid) { int oldnid; if (nid < 0) { /* just initialize by zero */ @@ -51,7 +51,7 @@ void __cpuinit map_cpu_to_node(int cpu, int nid) return; } -void __cpuinit unmap_cpu_from_node(int cpu, int nid) +void unmap_cpu_from_node(int cpu, int nid) { WARN_ON(!cpu_isset(cpu, node_to_cpu_mask[nid])); WARN_ON(cpu_to_node_map[cpu] != nid); diff --git a/arch/ia64/kernel/palinfo.c b/arch/ia64/kernel/palinfo.c index 2b3c2d79256f9..ab333284f4b2e 100644 --- a/arch/ia64/kernel/palinfo.c +++ b/arch/ia64/kernel/palinfo.c @@ -932,7 +932,7 @@ static const struct file_operations proc_palinfo_fops = { .release = single_release, }; -static void __cpuinit +static void create_palinfo_proc_entries(unsigned int cpu) { pal_func_cpu_u_t f; @@ -962,7 +962,7 @@ remove_palinfo_proc_entries(unsigned int hcpu) remove_proc_subtree(cpustr, palinfo_dir); } -static int __cpuinit palinfo_cpu_callback(struct notifier_block *nfb, +static int palinfo_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int hotcpu = (unsigned long)hcpu; diff --git a/arch/ia64/kernel/salinfo.c b/arch/ia64/kernel/salinfo.c index 4bc580af67b39..960a396f59296 100644 --- a/arch/ia64/kernel/salinfo.c +++ b/arch/ia64/kernel/salinfo.c @@ -568,7 +568,7 @@ static const struct file_operations salinfo_data_fops = { .llseek = default_llseek, }; -static int __cpuinit +static int salinfo_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu) { unsigned int i, cpu = (unsigned long)hcpu; @@ -609,7 +609,7 @@ salinfo_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu return NOTIFY_OK; } -static struct notifier_block salinfo_cpu_notifier __cpuinitdata = +static struct notifier_block salinfo_cpu_notifier = { .notifier_call = salinfo_cpu_callback, .priority = 0, diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c index 13bfdd22afc8e..fff7d768c669f 100644 --- a/arch/ia64/kernel/setup.c +++ b/arch/ia64/kernel/setup.c @@ -748,7 +748,7 @@ const struct seq_operations cpuinfo_op = { #define MAX_BRANDS 8 static char brandname[MAX_BRANDS][128]; -static char * __cpuinit +static char * get_model_name(__u8 family, __u8 model) { static int overflow; @@ -778,7 +778,7 @@ get_model_name(__u8 family, __u8 model) return "Unknown"; } -static void __cpuinit +static void identify_cpu (struct cpuinfo_ia64 *c) { union { @@ -850,7 +850,7 @@ identify_cpu (struct cpuinfo_ia64 *c) * 2. the minimum of the i-cache stride sizes for "flush_icache_range()". * 3. the minimum of the cache stride sizes for "clflush_cache_range()". */ -static void __cpuinit +static void get_cache_info(void) { unsigned long line_size, max = 1; @@ -915,10 +915,10 @@ get_cache_info(void) * cpu_init() initializes state that is per-CPU. This function acts * as a 'CPU state barrier', nothing should get across. */ -void __cpuinit +void cpu_init (void) { - extern void __cpuinit ia64_mmu_init (void *); + extern void ia64_mmu_init (void *); static unsigned long max_num_phys_stacked = IA64_NUM_PHYS_STACK_REG; unsigned long num_phys_stacked; pal_vm_info_2_u_t vmi; diff --git a/arch/ia64/kernel/smpboot.c b/arch/ia64/kernel/smpboot.c index 8d87168d218df..547a48d78bd7f 100644 --- a/arch/ia64/kernel/smpboot.c +++ b/arch/ia64/kernel/smpboot.c @@ -351,7 +351,7 @@ static inline void smp_setup_percpu_timer(void) { } -static void __cpuinit +static void smp_callin (void) { int cpuid, phys_id, itc_master; @@ -442,7 +442,7 @@ smp_callin (void) /* * Activate a secondary processor. head.S calls this. */ -int __cpuinit +int start_secondary (void *unused) { /* Early console may use I/O ports */ @@ -459,7 +459,7 @@ start_secondary (void *unused) return 0; } -static int __cpuinit +static int do_boot_cpu (int sapicid, int cpu, struct task_struct *idle) { int timeout; @@ -728,7 +728,7 @@ static inline void set_cpu_sibling_map(int cpu) } } -int __cpuinit +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { int ret; diff --git a/arch/ia64/kernel/topology.c b/arch/ia64/kernel/topology.c index dc00b2c1b42ab..33980354ee466 100644 --- a/arch/ia64/kernel/topology.c +++ b/arch/ia64/kernel/topology.c @@ -139,7 +139,7 @@ static struct cpu_cache_info all_cpu_cache_info[NR_CPUS] __cpuinitdata; #define LEAF_KOBJECT_PTR(x,y) (&all_cpu_cache_info[x].cache_leaves[y]) #ifdef CONFIG_SMP -static void __cpuinit cache_shared_cpu_map_setup( unsigned int cpu, +static void cache_shared_cpu_map_setup( unsigned int cpu, struct cache_info * this_leaf) { pal_cache_shared_info_t csi; @@ -174,7 +174,7 @@ static void __cpuinit cache_shared_cpu_map_setup( unsigned int cpu, &csi) == PAL_STATUS_SUCCESS); } #else -static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, +static void cache_shared_cpu_map_setup(unsigned int cpu, struct cache_info * this_leaf) { cpu_set(cpu, this_leaf->shared_cpu_map); @@ -298,7 +298,7 @@ static struct kobj_type cache_ktype_percpu_entry = { .sysfs_ops = &cache_sysfs_ops, }; -static void __cpuinit cpu_cache_sysfs_exit(unsigned int cpu) +static void cpu_cache_sysfs_exit(unsigned int cpu) { kfree(all_cpu_cache_info[cpu].cache_leaves); all_cpu_cache_info[cpu].cache_leaves = NULL; @@ -307,7 +307,7 @@ static void __cpuinit cpu_cache_sysfs_exit(unsigned int cpu) return; } -static int __cpuinit cpu_cache_sysfs_init(unsigned int cpu) +static int cpu_cache_sysfs_init(unsigned int cpu) { unsigned long i, levels, unique_caches; pal_cache_config_info_t cci; @@ -351,7 +351,7 @@ static int __cpuinit cpu_cache_sysfs_init(unsigned int cpu) } /* Add cache interface for CPU device */ -static int __cpuinit cache_add_dev(struct device * sys_dev) +static int cache_add_dev(struct device * sys_dev) { unsigned int cpu = sys_dev->id; unsigned long i, j; @@ -401,7 +401,7 @@ static int __cpuinit cache_add_dev(struct device * sys_dev) } /* Remove cache interface for CPU device */ -static int __cpuinit cache_remove_dev(struct device * sys_dev) +static int cache_remove_dev(struct device * sys_dev) { unsigned int cpu = sys_dev->id; unsigned long i; @@ -425,7 +425,7 @@ static int __cpuinit cache_remove_dev(struct device * sys_dev) * When a cpu is hot-plugged, do a check and initiate * cache kobject if necessary */ -static int __cpuinit cache_cpu_callback(struct notifier_block *nfb, +static int cache_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; @@ -445,7 +445,7 @@ static int __cpuinit cache_cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata cache_cpu_notifier = +static struct notifier_block cache_cpu_notifier = { .notifier_call = cache_cpu_callback }; diff --git a/arch/ia64/mm/contig.c b/arch/ia64/mm/contig.c index 67c59ebec899c..41d83fe5ee4de 100644 --- a/arch/ia64/mm/contig.c +++ b/arch/ia64/mm/contig.c @@ -156,7 +156,7 @@ static void *cpu_data; * * Allocate and setup per-cpu data areas. */ -void * __cpuinit +void * per_cpu_init (void) { static bool first_time = true; diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c index ae4db4bd6d97d..7253d83650bf8 100644 --- a/arch/ia64/mm/discontig.c +++ b/arch/ia64/mm/discontig.c @@ -593,7 +593,7 @@ void __init find_memory(void) * find_pernode_space() does most of this already, we just need to set * local_per_cpu_offset */ -void __cpuinit *per_cpu_init(void) +void *per_cpu_init(void) { int cpu; static int first_time = 1; diff --git a/arch/ia64/mm/numa.c b/arch/ia64/mm/numa.c index 4248492b9321b..ea21d4cad540e 100644 --- a/arch/ia64/mm/numa.c +++ b/arch/ia64/mm/numa.c @@ -86,7 +86,7 @@ int __meminit __early_pfn_to_nid(unsigned long pfn) return -1; } -void __cpuinit numa_clear_node(int cpu) +void numa_clear_node(int cpu) { unmap_cpu_from_node(cpu, NUMA_NO_NODE); } diff --git a/arch/ia64/sn/kernel/setup.c b/arch/ia64/sn/kernel/setup.c index f82e7b462b7b1..53b01b8e2f197 100644 --- a/arch/ia64/sn/kernel/setup.c +++ b/arch/ia64/sn/kernel/setup.c @@ -192,7 +192,7 @@ void __init early_sn_setup(void) } extern int platform_intr_list[]; -static int __cpuinitdata shub_1_1_found; +static int shub_1_1_found; /* * sn_check_for_wars @@ -200,7 +200,7 @@ static int __cpuinitdata shub_1_1_found; * Set flag for enabling shub specific wars */ -static inline int __cpuinit is_shub_1_1(int nasid) +static inline int is_shub_1_1(int nasid) { unsigned long id; int rev; @@ -212,7 +212,7 @@ static inline int __cpuinit is_shub_1_1(int nasid) return rev <= 2; } -static void __cpuinit sn_check_for_wars(void) +static void sn_check_for_wars(void) { int cnode; @@ -558,7 +558,7 @@ static void __init sn_init_pdas(char **cmdline_p) * Also sets up a few fields in the nodepda. Also known as * platform_cpu_init() by the ia64 machvec code. */ -void __cpuinit sn_cpu_init(void) +void sn_cpu_init(void) { int cpuid; int cpuphyid; diff --git a/arch/ia64/xen/hypervisor.c b/arch/ia64/xen/hypervisor.c index 52172eee85913..fab62528a80b4 100644 --- a/arch/ia64/xen/hypervisor.c +++ b/arch/ia64/xen/hypervisor.c @@ -74,7 +74,7 @@ void __init xen_setup_vcpu_info_placement(void) xen_vcpu_setup(cpu); } -void __cpuinit +void xen_cpu_init(void) { xen_smp_intr_init(); diff --git a/arch/m32r/kernel/smpboot.c b/arch/m32r/kernel/smpboot.c index 0ac558adc6055..bb21f4f631700 100644 --- a/arch/m32r/kernel/smpboot.c +++ b/arch/m32r/kernel/smpboot.c @@ -343,7 +343,7 @@ static void __init do_boot_cpu(int phys_id) } } -int __cpuinit __cpu_up(unsigned int cpu_id, struct task_struct *tidle) +int __cpu_up(unsigned int cpu_id, struct task_struct *tidle) { int timeout; diff --git a/arch/metag/kernel/perf/perf_event.c b/arch/metag/kernel/perf/perf_event.c index 366569425c52a..49ae6ff4bac69 100644 --- a/arch/metag/kernel/perf/perf_event.c +++ b/arch/metag/kernel/perf/perf_event.c @@ -813,7 +813,7 @@ static struct metag_pmu _metag_pmu = { }; /* PMU CPU hotplug notifier */ -static int __cpuinit metag_pmu_cpu_notify(struct notifier_block *b, +static int metag_pmu_cpu_notify(struct notifier_block *b, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned int)hcpu; @@ -828,7 +828,7 @@ static int __cpuinit metag_pmu_cpu_notify(struct notifier_block *b, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata metag_pmu_notifier = { +static struct notifier_block metag_pmu_notifier = { .notifier_call = metag_pmu_cpu_notify, }; diff --git a/arch/metag/kernel/smp.c b/arch/metag/kernel/smp.c index f443ec9a7cbef..6f9d5ac915c76 100644 --- a/arch/metag/kernel/smp.c +++ b/arch/metag/kernel/smp.c @@ -65,7 +65,7 @@ static DEFINE_SPINLOCK(boot_lock); /* * "thread" is assumed to be a valid Meta hardware thread ID. */ -int __cpuinit boot_secondary(unsigned int thread, struct task_struct *idle) +int boot_secondary(unsigned int thread, struct task_struct *idle) { u32 val; @@ -115,7 +115,7 @@ int __cpuinit boot_secondary(unsigned int thread, struct task_struct *idle) * If the cache partition has changed, prints a message to the log describing * those changes. */ -static __cpuinit void describe_cachepart_change(unsigned int thread, +static void describe_cachepart_change(unsigned int thread, const char *label, unsigned int sz, unsigned int old, @@ -167,7 +167,7 @@ static __cpuinit void describe_cachepart_change(unsigned int thread, * Ensures that coherency is enabled and that the threads share the same cache * partitions. */ -static __cpuinit void setup_smp_cache(unsigned int thread) +static void setup_smp_cache(unsigned int thread) { unsigned int this_thread, lflags; unsigned int dcsz, dcpart_this, dcpart_old, dcpart_new; @@ -212,7 +212,7 @@ static __cpuinit void setup_smp_cache(unsigned int thread) icpart_old, icpart_new); } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) +int __cpu_up(unsigned int cpu, struct task_struct *idle) { unsigned int thread = cpu_2_hwthread_id[cpu]; int ret; @@ -273,7 +273,7 @@ static DECLARE_COMPLETION(cpu_killed); /* * __cpu_disable runs on the processor to be shutdown. */ -int __cpuexit __cpu_disable(void) +int __cpu_disable(void) { unsigned int cpu = smp_processor_id(); struct task_struct *p; @@ -310,7 +310,7 @@ int __cpuexit __cpu_disable(void) * called on the thread which is asking for a CPU to be shutdown - * waits until shutdown has completed, or it is timed out. */ -void __cpuexit __cpu_die(unsigned int cpu) +void __cpu_die(unsigned int cpu) { if (!wait_for_completion_timeout(&cpu_killed, msecs_to_jiffies(1))) pr_err("CPU%u: unable to kill\n", cpu); @@ -322,7 +322,7 @@ void __cpuexit __cpu_die(unsigned int cpu) * Note that we do not return from this function. If this cpu is * brought online again it will need to run secondary_startup(). */ -void __cpuexit cpu_die(void) +void cpu_die(void) { local_irq_disable(); idle_task_exit(); @@ -337,7 +337,7 @@ void __cpuexit cpu_die(void) * Called by both boot and secondaries to move global data into * per-processor storage. */ -void __cpuinit smp_store_cpu_info(unsigned int cpuid) +void smp_store_cpu_info(unsigned int cpuid) { struct cpuinfo_metag *cpu_info = &per_cpu(cpu_data, cpuid); diff --git a/arch/metag/kernel/traps.c b/arch/metag/kernel/traps.c index 2ceeaae5b1992..a645da359a3f4 100644 --- a/arch/metag/kernel/traps.c +++ b/arch/metag/kernel/traps.c @@ -811,7 +811,7 @@ static void set_trigger_mask(unsigned int mask) } #endif -void __cpuinit per_cpu_trap_init(unsigned long cpu) +void per_cpu_trap_init(unsigned long cpu) { TBIRES int_context; unsigned int thread = cpu_2_hwthread_id[cpu]; diff --git a/arch/mips/ath79/setup.c b/arch/mips/ath79/setup.c index 8be4e856b8b89..80f4ecd42b0dc 100644 --- a/arch/mips/ath79/setup.c +++ b/arch/mips/ath79/setup.c @@ -182,7 +182,7 @@ const char *get_system_type(void) return ath79_sys_type; } -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { return CP0_LEGACY_COMPARE_IRQ; } diff --git a/arch/mips/cavium-octeon/octeon-irq.c b/arch/mips/cavium-octeon/octeon-irq.c index 45c1a6caa2066..5e3da374caab1 100644 --- a/arch/mips/cavium-octeon/octeon-irq.c +++ b/arch/mips/cavium-octeon/octeon-irq.c @@ -1095,7 +1095,7 @@ static void octeon_irq_ip3_ciu(void) static bool octeon_irq_use_ip4; -static void __cpuinit octeon_irq_local_enable_ip4(void *arg) +static void octeon_irq_local_enable_ip4(void *arg) { set_c0_status(STATUSF_IP4); } @@ -1110,21 +1110,21 @@ static void (*octeon_irq_ip2)(void); static void (*octeon_irq_ip3)(void); static void (*octeon_irq_ip4)(void); -void __cpuinitdata (*octeon_irq_setup_secondary)(void); +void (*octeon_irq_setup_secondary)(void); -void __cpuinit octeon_irq_set_ip4_handler(octeon_irq_ip4_handler_t h) +void octeon_irq_set_ip4_handler(octeon_irq_ip4_handler_t h) { octeon_irq_ip4 = h; octeon_irq_use_ip4 = true; on_each_cpu(octeon_irq_local_enable_ip4, NULL, 1); } -static void __cpuinit octeon_irq_percpu_enable(void) +static void octeon_irq_percpu_enable(void) { irq_cpu_online(); } -static void __cpuinit octeon_irq_init_ciu_percpu(void) +static void octeon_irq_init_ciu_percpu(void) { int coreid = cvmx_get_core_num(); @@ -1167,7 +1167,7 @@ static void octeon_irq_init_ciu2_percpu(void) cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(coreid)); } -static void __cpuinit octeon_irq_setup_secondary_ciu(void) +static void octeon_irq_setup_secondary_ciu(void) { octeon_irq_init_ciu_percpu(); octeon_irq_percpu_enable(); diff --git a/arch/mips/cavium-octeon/smp.c b/arch/mips/cavium-octeon/smp.c index 295137dfdc37c..138cc80c59281 100644 --- a/arch/mips/cavium-octeon/smp.c +++ b/arch/mips/cavium-octeon/smp.c @@ -173,7 +173,7 @@ static void octeon_boot_secondary(int cpu, struct task_struct *idle) * After we've done initial boot, this function is called to allow the * board code to clean up state, if needed */ -static void __cpuinit octeon_init_secondary(void) +static void octeon_init_secondary(void) { unsigned int sr; @@ -375,7 +375,7 @@ static int octeon_update_boot_vector(unsigned int cpu) return 0; } -static int __cpuinit octeon_cpu_callback(struct notifier_block *nfb, +static int octeon_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; @@ -394,7 +394,7 @@ static int __cpuinit octeon_cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static int __cpuinit register_cavium_notifier(void) +static int register_cavium_notifier(void) { hotcpu_notifier(octeon_cpu_callback, 0); return 0; diff --git a/arch/mips/include/asm/uasm.h b/arch/mips/include/asm/uasm.h index 370d967725c28..dc4436e7132f5 100644 --- a/arch/mips/include/asm/uasm.h +++ b/arch/mips/include/asm/uasm.h @@ -17,8 +17,8 @@ #define __uasminitdata #define UASM_EXPORT_SYMBOL(sym) EXPORT_SYMBOL(sym) #else -#define __uasminit __cpuinit -#define __uasminitdata __cpuinitdata +#define __uasminit +#define __uasminitdatadata #define UASM_EXPORT_SYMBOL(sym) #endif diff --git a/arch/mips/kernel/cevt-bcm1480.c b/arch/mips/kernel/cevt-bcm1480.c index 15f618b40cf6c..7976457184b18 100644 --- a/arch/mips/kernel/cevt-bcm1480.c +++ b/arch/mips/kernel/cevt-bcm1480.c @@ -109,7 +109,7 @@ static DEFINE_PER_CPU(struct clock_event_device, sibyte_hpt_clockevent); static DEFINE_PER_CPU(struct irqaction, sibyte_hpt_irqaction); static DEFINE_PER_CPU(char [18], sibyte_hpt_name); -void __cpuinit sb1480_clockevent_init(void) +void sb1480_clockevent_init(void) { unsigned int cpu = smp_processor_id(); unsigned int irq = K_BCM1480_INT_TIMER_0 + cpu; diff --git a/arch/mips/kernel/cevt-gic.c b/arch/mips/kernel/cevt-gic.c index 730eaf92c0189..594cbbf16d627 100644 --- a/arch/mips/kernel/cevt-gic.c +++ b/arch/mips/kernel/cevt-gic.c @@ -59,7 +59,7 @@ void gic_event_handler(struct clock_event_device *dev) { } -int __cpuinit gic_clockevent_init(void) +int gic_clockevent_init(void) { unsigned int cpu = smp_processor_id(); struct clock_event_device *cd; diff --git a/arch/mips/kernel/cevt-r4k.c b/arch/mips/kernel/cevt-r4k.c index 02033eaf88254..50d3f5a8d6bb1 100644 --- a/arch/mips/kernel/cevt-r4k.c +++ b/arch/mips/kernel/cevt-r4k.c @@ -171,7 +171,7 @@ int c0_compare_int_usable(void) } #ifndef CONFIG_MIPS_MT_SMTC -int __cpuinit r4k_clockevent_init(void) +int r4k_clockevent_init(void) { unsigned int cpu = smp_processor_id(); struct clock_event_device *cd; diff --git a/arch/mips/kernel/cevt-sb1250.c b/arch/mips/kernel/cevt-sb1250.c index 200f2778bf366..5ea6d6b1de150 100644 --- a/arch/mips/kernel/cevt-sb1250.c +++ b/arch/mips/kernel/cevt-sb1250.c @@ -107,7 +107,7 @@ static DEFINE_PER_CPU(struct clock_event_device, sibyte_hpt_clockevent); static DEFINE_PER_CPU(struct irqaction, sibyte_hpt_irqaction); static DEFINE_PER_CPU(char [18], sibyte_hpt_name); -void __cpuinit sb1250_clockevent_init(void) +void sb1250_clockevent_init(void) { unsigned int cpu = smp_processor_id(); unsigned int irq = K_INT_TIMER_0 + cpu; diff --git a/arch/mips/kernel/cevt-smtc.c b/arch/mips/kernel/cevt-smtc.c index 9de5ed7ef1a36..b6cf0a60d8968 100644 --- a/arch/mips/kernel/cevt-smtc.c +++ b/arch/mips/kernel/cevt-smtc.c @@ -248,7 +248,7 @@ irqreturn_t c0_compare_interrupt(int irq, void *dev_id) } -int __cpuinit smtc_clockevent_init(void) +int smtc_clockevent_init(void) { uint64_t mips_freq = mips_hpt_frequency; unsigned int cpu = smp_processor_id(); diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index c6568bf4b1b05..d866a45b551f2 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -27,7 +27,7 @@ #include #include -static int __cpuinitdata mips_fpu_disabled; +static int mips_fpu_disabled; static int __init fpu_disable(char *s) { @@ -39,7 +39,7 @@ static int __init fpu_disable(char *s) __setup("nofpu", fpu_disable); -int __cpuinitdata mips_dsp_disabled; +int mips_dsp_disabled; static int __init dsp_disable(char *s) { @@ -134,7 +134,7 @@ static inline void cpu_probe_vmbits(struct cpuinfo_mips *c) #endif } -static void __cpuinit set_isa(struct cpuinfo_mips *c, unsigned int isa) +static void set_isa(struct cpuinfo_mips *c, unsigned int isa) { switch (isa) { case MIPS_CPU_ISA_M64R2: @@ -162,7 +162,7 @@ static void __cpuinit set_isa(struct cpuinfo_mips *c, unsigned int isa) } } -static char unknown_isa[] __cpuinitdata = KERN_ERR \ +static char unknown_isa[] = KERN_ERR \ "Unsupported ISA type, c0.config0: %d."; static inline unsigned int decode_config0(struct cpuinfo_mips *c) @@ -296,7 +296,7 @@ static inline unsigned int decode_config4(struct cpuinfo_mips *c) return config4 & MIPS_CONF_M; } -static void __cpuinit decode_configs(struct cpuinfo_mips *c) +static void decode_configs(struct cpuinfo_mips *c) { int ok; @@ -970,7 +970,7 @@ EXPORT_SYMBOL(__ua_limit); const char *__cpu_name[NR_CPUS]; const char *__elf_platform; -__cpuinit void cpu_probe(void) +void cpu_probe(void) { struct cpuinfo_mips *c = ¤t_cpu_data; unsigned int cpu = smp_processor_id(); @@ -1055,7 +1055,7 @@ __cpuinit void cpu_probe(void) #endif } -__cpuinit void cpu_report(void) +void cpu_report(void) { struct cpuinfo_mips *c = ¤t_cpu_data; diff --git a/arch/mips/kernel/smp-bmips.c b/arch/mips/kernel/smp-bmips.c index 8e393b8443f7a..20c637fdae892 100644 --- a/arch/mips/kernel/smp-bmips.c +++ b/arch/mips/kernel/smp-bmips.c @@ -381,7 +381,7 @@ struct plat_smp_ops bmips_smp_ops = { * UP BMIPS systems as well. ***********************************************************************/ -static void __cpuinit bmips_wr_vec(unsigned long dst, char *start, char *end) +static void bmips_wr_vec(unsigned long dst, char *start, char *end) { memcpy((void *)dst, start, end - start); dma_cache_wback((unsigned long)start, end - start); @@ -389,7 +389,7 @@ static void __cpuinit bmips_wr_vec(unsigned long dst, char *start, char *end) instruction_hazard(); } -static inline void __cpuinit bmips_nmi_handler_setup(void) +static inline void bmips_nmi_handler_setup(void) { bmips_wr_vec(BMIPS_NMI_RESET_VEC, &bmips_reset_nmi_vec, &bmips_reset_nmi_vec_end); @@ -397,7 +397,7 @@ static inline void __cpuinit bmips_nmi_handler_setup(void) &bmips_smp_int_vec_end); } -void __cpuinit bmips_ebase_setup(void) +void bmips_ebase_setup(void) { unsigned long new_ebase = ebase; void __iomem __maybe_unused *cbr; diff --git a/arch/mips/kernel/smp-mt.c b/arch/mips/kernel/smp-mt.c index 3e5164c11caca..57a3f7a2b370c 100644 --- a/arch/mips/kernel/smp-mt.c +++ b/arch/mips/kernel/smp-mt.c @@ -149,7 +149,7 @@ static void vsmp_send_ipi_mask(const struct cpumask *mask, unsigned int action) vsmp_send_ipi_single(i, action); } -static void __cpuinit vsmp_init_secondary(void) +static void vsmp_init_secondary(void) { #ifdef CONFIG_IRQ_GIC /* This is Malta specific: IPI,performance and timer interrupts */ @@ -162,7 +162,7 @@ static void __cpuinit vsmp_init_secondary(void) STATUSF_IP6 | STATUSF_IP7); } -static void __cpuinit vsmp_smp_finish(void) +static void vsmp_smp_finish(void) { /* CDFIXME: remove this? */ write_c0_compare(read_c0_count() + (8* mips_hpt_frequency/HZ)); @@ -188,7 +188,7 @@ static void vsmp_cpus_done(void) * (unsigned long)idle->thread_info the gp * assumes a 1:1 mapping of TC => VPE */ -static void __cpuinit vsmp_boot_secondary(int cpu, struct task_struct *idle) +static void vsmp_boot_secondary(int cpu, struct task_struct *idle) { struct thread_info *gp = task_thread_info(idle); dvpe(); diff --git a/arch/mips/kernel/smp-up.c b/arch/mips/kernel/smp-up.c index 00500fea27507..7fde3e4d978f9 100644 --- a/arch/mips/kernel/smp-up.c +++ b/arch/mips/kernel/smp-up.c @@ -28,11 +28,11 @@ static inline void up_send_ipi_mask(const struct cpumask *mask, * After we've done initial boot, this function is called to allow the * board code to clean up state, if needed */ -static void __cpuinit up_init_secondary(void) +static void up_init_secondary(void) { } -static void __cpuinit up_smp_finish(void) +static void up_smp_finish(void) { } @@ -44,7 +44,7 @@ static void up_cpus_done(void) /* * Firmware CPU startup hook */ -static void __cpuinit up_boot_secondary(int cpu, struct task_struct *idle) +static void up_boot_secondary(int cpu, struct task_struct *idle) { } diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c index caeec21df1c4b..57b89cba16245 100644 --- a/arch/mips/kernel/smp.c +++ b/arch/mips/kernel/smp.c @@ -86,7 +86,7 @@ static inline void set_cpu_sibling_map(int cpu) struct plat_smp_ops *mp_ops; EXPORT_SYMBOL(mp_ops); -__cpuinit void register_smp_ops(struct plat_smp_ops *ops) +void register_smp_ops(struct plat_smp_ops *ops) { if (mp_ops) printk(KERN_WARNING "Overriding previously set SMP ops\n"); @@ -98,7 +98,7 @@ __cpuinit void register_smp_ops(struct plat_smp_ops *ops) * First C code run on the secondary CPUs after being started up by * the master. */ -asmlinkage __cpuinit void start_secondary(void) +asmlinkage void start_secondary(void) { unsigned int cpu; @@ -197,7 +197,7 @@ void smp_prepare_boot_cpu(void) cpu_set(0, cpu_callin_map); } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { mp_ops->boot_secondary(cpu, tidle); diff --git a/arch/mips/kernel/smtc.c b/arch/mips/kernel/smtc.c index 75a4fd709841a..dfc1b911be04b 100644 --- a/arch/mips/kernel/smtc.c +++ b/arch/mips/kernel/smtc.c @@ -645,7 +645,7 @@ void smtc_prepare_cpus(int cpus) * (unsigned long)idle->thread_info the gp * */ -void __cpuinit smtc_boot_secondary(int cpu, struct task_struct *idle) +void smtc_boot_secondary(int cpu, struct task_struct *idle) { extern u32 kernelsp[NR_CPUS]; unsigned long flags; diff --git a/arch/mips/kernel/spram.c b/arch/mips/kernel/spram.c index 6af08d896e20b..93f86817f20a6 100644 --- a/arch/mips/kernel/spram.c +++ b/arch/mips/kernel/spram.c @@ -37,7 +37,7 @@ /* * Different semantics to the set_c0_* function built by __BUILD_SET_C0 */ -static __cpuinit unsigned int bis_c0_errctl(unsigned int set) +static unsigned int bis_c0_errctl(unsigned int set) { unsigned int res; res = read_c0_errctl(); @@ -45,7 +45,7 @@ static __cpuinit unsigned int bis_c0_errctl(unsigned int set) return res; } -static __cpuinit void ispram_store_tag(unsigned int offset, unsigned int data) +static void ispram_store_tag(unsigned int offset, unsigned int data) { unsigned int errctl; @@ -64,7 +64,7 @@ static __cpuinit void ispram_store_tag(unsigned int offset, unsigned int data) } -static __cpuinit unsigned int ispram_load_tag(unsigned int offset) +static unsigned int ispram_load_tag(unsigned int offset) { unsigned int data; unsigned int errctl; @@ -82,7 +82,7 @@ static __cpuinit unsigned int ispram_load_tag(unsigned int offset) return data; } -static __cpuinit void dspram_store_tag(unsigned int offset, unsigned int data) +static void dspram_store_tag(unsigned int offset, unsigned int data) { unsigned int errctl; @@ -98,7 +98,7 @@ static __cpuinit void dspram_store_tag(unsigned int offset, unsigned int data) } -static __cpuinit unsigned int dspram_load_tag(unsigned int offset) +static unsigned int dspram_load_tag(unsigned int offset) { unsigned int data; unsigned int errctl; @@ -115,7 +115,7 @@ static __cpuinit unsigned int dspram_load_tag(unsigned int offset) return data; } -static __cpuinit void probe_spram(char *type, +static void probe_spram(char *type, unsigned int base, unsigned int (*read)(unsigned int), void (*write)(unsigned int, unsigned int)) @@ -196,7 +196,7 @@ static __cpuinit void probe_spram(char *type, offset += 2 * SPRAM_TAG_STRIDE; } } -void __cpuinit spram_config(void) +void spram_config(void) { struct cpuinfo_mips *c = ¤t_cpu_data; unsigned int config0; diff --git a/arch/mips/kernel/sync-r4k.c b/arch/mips/kernel/sync-r4k.c index 1ff43d5ac2c4e..84536bf4a1540 100644 --- a/arch/mips/kernel/sync-r4k.c +++ b/arch/mips/kernel/sync-r4k.c @@ -20,15 +20,15 @@ #include #include -static atomic_t __cpuinitdata count_start_flag = ATOMIC_INIT(0); -static atomic_t __cpuinitdata count_count_start = ATOMIC_INIT(0); -static atomic_t __cpuinitdata count_count_stop = ATOMIC_INIT(0); -static atomic_t __cpuinitdata count_reference = ATOMIC_INIT(0); +static atomic_t count_start_flag = ATOMIC_INIT(0); +static atomic_t count_count_start = ATOMIC_INIT(0); +static atomic_t count_count_stop = ATOMIC_INIT(0); +static atomic_t count_reference = ATOMIC_INIT(0); #define COUNTON 100 #define NR_LOOPS 5 -void __cpuinit synchronise_count_master(int cpu) +void synchronise_count_master(int cpu) { int i; unsigned long flags; @@ -106,7 +106,7 @@ void __cpuinit synchronise_count_master(int cpu) printk("done.\n"); } -void __cpuinit synchronise_count_slave(int cpu) +void synchronise_count_slave(int cpu) { int i; unsigned int initcount; diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index a75ae40184aa3..4846c221ceec7 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -1642,7 +1642,7 @@ int cp0_compare_irq_shift; int cp0_perfcount_irq; EXPORT_SYMBOL_GPL(cp0_perfcount_irq); -static int __cpuinitdata noulri; +static int noulri; static int __init ulri_disable(char *s) { @@ -1653,7 +1653,7 @@ static int __init ulri_disable(char *s) } __setup("noulri", ulri_disable); -void __cpuinit per_cpu_trap_init(bool is_boot_cpu) +void per_cpu_trap_init(bool is_boot_cpu) { unsigned int cpu = smp_processor_id(); unsigned int status_set = ST0_CU0; @@ -1770,7 +1770,7 @@ void __cpuinit per_cpu_trap_init(bool is_boot_cpu) } /* Install CPU exception handler */ -void __cpuinit set_handler(unsigned long offset, void *addr, unsigned long size) +void set_handler(unsigned long offset, void *addr, unsigned long size) { #ifdef CONFIG_CPU_MICROMIPS memcpy((void *)(ebase + offset), ((unsigned char *)addr - 1), size); @@ -1780,7 +1780,7 @@ void __cpuinit set_handler(unsigned long offset, void *addr, unsigned long size) local_flush_icache_range(ebase + offset, ebase + offset + size); } -static char panic_null_cerr[] __cpuinitdata = +static char panic_null_cerr[] = "Trying to set NULL cache error exception handler"; /* @@ -1788,7 +1788,7 @@ static char panic_null_cerr[] __cpuinitdata = * This is suitable only for the cache error exception which is the only * exception handler that is being run uncached. */ -void __cpuinit set_uncached_handler(unsigned long offset, void *addr, +void set_uncached_handler(unsigned long offset, void *addr, unsigned long size) { unsigned long uncached_ebase = CKSEG1ADDR(ebase); diff --git a/arch/mips/kernel/watch.c b/arch/mips/kernel/watch.c index 7726f6157d9e7..3b7b9003ae2aa 100644 --- a/arch/mips/kernel/watch.c +++ b/arch/mips/kernel/watch.c @@ -100,7 +100,7 @@ void mips_clear_watch_registers(void) } } -__cpuinit void mips_probe_watch_registers(struct cpuinfo_mips *c) +void mips_probe_watch_registers(struct cpuinfo_mips *c) { unsigned int t; diff --git a/arch/mips/lantiq/irq.c b/arch/mips/lantiq/irq.c index 51194875f1582..eb3e186596304 100644 --- a/arch/mips/lantiq/irq.c +++ b/arch/mips/lantiq/irq.c @@ -461,7 +461,7 @@ int __init icu_of_init(struct device_node *node, struct device_node *parent) return 0; } -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { return MIPS_CPU_TIMER_IRQ; } diff --git a/arch/mips/lib/uncached.c b/arch/mips/lib/uncached.c index 65e3dfc4e5855..d8522f8e842a2 100644 --- a/arch/mips/lib/uncached.c +++ b/arch/mips/lib/uncached.c @@ -36,7 +36,7 @@ * values, so we can avoid sharing the same stack area between a cached * and the uncached mode. */ -unsigned long __cpuinit run_uncached(void *func) +unsigned long run_uncached(void *func) { register long sp __asm__("$sp"); register long ret __asm__("$2"); diff --git a/arch/mips/mm/c-octeon.c b/arch/mips/mm/c-octeon.c index 8557fb5528632..a0bcdbb81d410 100644 --- a/arch/mips/mm/c-octeon.c +++ b/arch/mips/mm/c-octeon.c @@ -180,7 +180,7 @@ static void octeon_flush_kernel_vmap_range(unsigned long vaddr, int size) * Probe Octeon's caches * */ -static void __cpuinit probe_octeon(void) +static void probe_octeon(void) { unsigned long icache_size; unsigned long dcache_size; @@ -251,7 +251,7 @@ static void __cpuinit probe_octeon(void) } } -static void __cpuinit octeon_cache_error_setup(void) +static void octeon_cache_error_setup(void) { extern char except_vec2_octeon; set_handler(0x100, &except_vec2_octeon, 0x80); @@ -261,7 +261,7 @@ static void __cpuinit octeon_cache_error_setup(void) * Setup the Octeon cache flush routines * */ -void __cpuinit octeon_cache_init(void) +void octeon_cache_init(void) { probe_octeon(); diff --git a/arch/mips/mm/c-r3k.c b/arch/mips/mm/c-r3k.c index 704dc735a59df..2fcde0c8ea029 100644 --- a/arch/mips/mm/c-r3k.c +++ b/arch/mips/mm/c-r3k.c @@ -26,7 +26,7 @@ static unsigned long icache_size, dcache_size; /* Size in bytes */ static unsigned long icache_lsize, dcache_lsize; /* Size in bytes */ -unsigned long __cpuinit r3k_cache_size(unsigned long ca_flags) +unsigned long r3k_cache_size(unsigned long ca_flags) { unsigned long flags, status, dummy, size; volatile unsigned long *p; @@ -61,7 +61,7 @@ unsigned long __cpuinit r3k_cache_size(unsigned long ca_flags) return size * sizeof(*p); } -unsigned long __cpuinit r3k_cache_lsize(unsigned long ca_flags) +unsigned long r3k_cache_lsize(unsigned long ca_flags) { unsigned long flags, status, lsize, i; volatile unsigned long *p; @@ -90,7 +90,7 @@ unsigned long __cpuinit r3k_cache_lsize(unsigned long ca_flags) return lsize * sizeof(*p); } -static void __cpuinit r3k_probe_cache(void) +static void r3k_probe_cache(void) { dcache_size = r3k_cache_size(ST0_ISC); if (dcache_size) @@ -312,7 +312,7 @@ static void r3k_dma_cache_wback_inv(unsigned long start, unsigned long size) r3k_flush_dcache_range(start, start + size); } -void __cpuinit r3k_cache_init(void) +void r3k_cache_init(void) { extern void build_clear_page(void); extern void build_copy_page(void); diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index c2ec87e5d1cc6..e48a1e5240ee5 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -108,7 +108,7 @@ static inline void r4k_blast_dcache_page_dc64(unsigned long addr) blast_dcache64_page(addr); } -static void __cpuinit r4k_blast_dcache_page_setup(void) +static void r4k_blast_dcache_page_setup(void) { unsigned long dc_lsize = cpu_dcache_line_size(); @@ -124,7 +124,7 @@ static void __cpuinit r4k_blast_dcache_page_setup(void) static void (* r4k_blast_dcache_page_indexed)(unsigned long addr); -static void __cpuinit r4k_blast_dcache_page_indexed_setup(void) +static void r4k_blast_dcache_page_indexed_setup(void) { unsigned long dc_lsize = cpu_dcache_line_size(); @@ -141,7 +141,7 @@ static void __cpuinit r4k_blast_dcache_page_indexed_setup(void) void (* r4k_blast_dcache)(void); EXPORT_SYMBOL(r4k_blast_dcache); -static void __cpuinit r4k_blast_dcache_setup(void) +static void r4k_blast_dcache_setup(void) { unsigned long dc_lsize = cpu_dcache_line_size(); @@ -228,7 +228,7 @@ static inline void tx49_blast_icache32_page_indexed(unsigned long page) static void (* r4k_blast_icache_page)(unsigned long addr); -static void __cpuinit r4k_blast_icache_page_setup(void) +static void r4k_blast_icache_page_setup(void) { unsigned long ic_lsize = cpu_icache_line_size(); @@ -245,7 +245,7 @@ static void __cpuinit r4k_blast_icache_page_setup(void) static void (* r4k_blast_icache_page_indexed)(unsigned long addr); -static void __cpuinit r4k_blast_icache_page_indexed_setup(void) +static void r4k_blast_icache_page_indexed_setup(void) { unsigned long ic_lsize = cpu_icache_line_size(); @@ -270,7 +270,7 @@ static void __cpuinit r4k_blast_icache_page_indexed_setup(void) void (* r4k_blast_icache)(void); EXPORT_SYMBOL(r4k_blast_icache); -static void __cpuinit r4k_blast_icache_setup(void) +static void r4k_blast_icache_setup(void) { unsigned long ic_lsize = cpu_icache_line_size(); @@ -291,7 +291,7 @@ static void __cpuinit r4k_blast_icache_setup(void) static void (* r4k_blast_scache_page)(unsigned long addr); -static void __cpuinit r4k_blast_scache_page_setup(void) +static void r4k_blast_scache_page_setup(void) { unsigned long sc_lsize = cpu_scache_line_size(); @@ -309,7 +309,7 @@ static void __cpuinit r4k_blast_scache_page_setup(void) static void (* r4k_blast_scache_page_indexed)(unsigned long addr); -static void __cpuinit r4k_blast_scache_page_indexed_setup(void) +static void r4k_blast_scache_page_indexed_setup(void) { unsigned long sc_lsize = cpu_scache_line_size(); @@ -327,7 +327,7 @@ static void __cpuinit r4k_blast_scache_page_indexed_setup(void) static void (* r4k_blast_scache)(void); -static void __cpuinit r4k_blast_scache_setup(void) +static void r4k_blast_scache_setup(void) { unsigned long sc_lsize = cpu_scache_line_size(); @@ -804,11 +804,11 @@ static inline void alias_74k_erratum(struct cpuinfo_mips *c) } } -static char *way_string[] __cpuinitdata = { NULL, "direct mapped", "2-way", +static char *way_string[] = { NULL, "direct mapped", "2-way", "3-way", "4-way", "5-way", "6-way", "7-way", "8-way" }; -static void __cpuinit probe_pcache(void) +static void probe_pcache(void) { struct cpuinfo_mips *c = ¤t_cpu_data; unsigned int config = read_c0_config(); @@ -1126,7 +1126,7 @@ static void __cpuinit probe_pcache(void) * executes in KSEG1 space or else you will crash and burn badly. You have * been warned. */ -static int __cpuinit probe_scache(void) +static int probe_scache(void) { unsigned long flags, addr, begin, end, pow2; unsigned int config = read_c0_config(); @@ -1203,7 +1203,7 @@ extern int r5k_sc_init(void); extern int rm7k_sc_init(void); extern int mips_sc_init(void); -static void __cpuinit setup_scache(void) +static void setup_scache(void) { struct cpuinfo_mips *c = ¤t_cpu_data; unsigned int config = read_c0_config(); @@ -1336,7 +1336,7 @@ static void nxp_pr4450_fixup_config(void) NXP_BARRIER(); } -static int __cpuinitdata cca = -1; +static int cca = -1; static int __init cca_setup(char *str) { @@ -1347,7 +1347,7 @@ static int __init cca_setup(char *str) early_param("cca", cca_setup); -static void __cpuinit coherency_setup(void) +static void coherency_setup(void) { if (cca < 0 || cca > 7) cca = read_c0_config() & CONF_CM_CMASK; @@ -1387,7 +1387,7 @@ static void __cpuinit coherency_setup(void) } } -static void __cpuinit r4k_cache_error_setup(void) +static void r4k_cache_error_setup(void) { extern char __weak except_vec2_generic; extern char __weak except_vec2_sb1; @@ -1405,7 +1405,7 @@ static void __cpuinit r4k_cache_error_setup(void) } } -void __cpuinit r4k_cache_init(void) +void r4k_cache_init(void) { extern void build_clear_page(void); extern void build_copy_page(void); diff --git a/arch/mips/mm/c-tx39.c b/arch/mips/mm/c-tx39.c index ba9da270289fa..8d909dbbf37f5 100644 --- a/arch/mips/mm/c-tx39.c +++ b/arch/mips/mm/c-tx39.c @@ -344,7 +344,7 @@ static __init void tx39_probe_cache(void) } } -void __cpuinit tx39_cache_init(void) +void tx39_cache_init(void) { extern void build_clear_page(void); extern void build_copy_page(void); diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c index 5aeb3eb0b72f8..15f813c303b45 100644 --- a/arch/mips/mm/cache.c +++ b/arch/mips/mm/cache.c @@ -182,7 +182,7 @@ static inline void setup_protection_map(void) } } -void __cpuinit cpu_cache_init(void) +void cpu_cache_init(void) { if (cpu_has_3k_cache) { extern void __weak r3k_cache_init(void); diff --git a/arch/mips/mm/page.c b/arch/mips/mm/page.c index 4eb8dcfaf1ce1..e5506a54959cd 100644 --- a/arch/mips/mm/page.c +++ b/arch/mips/mm/page.c @@ -66,8 +66,8 @@ UASM_L_LA(_copy_pref_both) UASM_L_LA(_copy_pref_store) /* We need one branch and therefore one relocation per target label. */ -static struct uasm_label __cpuinitdata labels[5]; -static struct uasm_reloc __cpuinitdata relocs[5]; +static struct uasm_label labels[5]; +static struct uasm_reloc relocs[5]; #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) @@ -88,7 +88,7 @@ static int half_copy_loop_size __cpuinitdata; static int cache_line_size __cpuinitdata; #define cache_line_mask() (cache_line_size - 1) -static inline void __cpuinit +static inline void pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off) { if (cpu_has_64bit_gp_regs && DADDI_WAR && r4k_daddiu_bug()) { @@ -108,7 +108,7 @@ pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off) } } -static void __cpuinit set_prefetch_parameters(void) +static void set_prefetch_parameters(void) { if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) clear_word_size = 8; @@ -199,7 +199,7 @@ static void __cpuinit set_prefetch_parameters(void) 4 * copy_word_size)); } -static void __cpuinit build_clear_store(u32 **buf, int off) +static void build_clear_store(u32 **buf, int off) { if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) { uasm_i_sd(buf, ZERO, off, A0); @@ -208,7 +208,7 @@ static void __cpuinit build_clear_store(u32 **buf, int off) } } -static inline void __cpuinit build_clear_pref(u32 **buf, int off) +static inline void build_clear_pref(u32 **buf, int off) { if (off & cache_line_mask()) return; @@ -240,7 +240,7 @@ extern u32 __clear_page_end; extern u32 __copy_page_start; extern u32 __copy_page_end; -void __cpuinit build_clear_page(void) +void build_clear_page(void) { int off; u32 *buf = &__clear_page_start; @@ -333,7 +333,7 @@ void __cpuinit build_clear_page(void) pr_debug("\t.set pop\n"); } -static void __cpuinit build_copy_load(u32 **buf, int reg, int off) +static void build_copy_load(u32 **buf, int reg, int off) { if (cpu_has_64bit_gp_regs) { uasm_i_ld(buf, reg, off, A1); @@ -342,7 +342,7 @@ static void __cpuinit build_copy_load(u32 **buf, int reg, int off) } } -static void __cpuinit build_copy_store(u32 **buf, int reg, int off) +static void build_copy_store(u32 **buf, int reg, int off) { if (cpu_has_64bit_gp_regs) { uasm_i_sd(buf, reg, off, A0); @@ -387,7 +387,7 @@ static inline void build_copy_store_pref(u32 **buf, int off) } } -void __cpuinit build_copy_page(void) +void build_copy_page(void) { int off; u32 *buf = &__copy_page_start; diff --git a/arch/mips/mm/sc-ip22.c b/arch/mips/mm/sc-ip22.c index c6aaed934d536..dc7c5a5214a9e 100644 --- a/arch/mips/mm/sc-ip22.c +++ b/arch/mips/mm/sc-ip22.c @@ -167,7 +167,7 @@ static struct bcache_ops indy_sc_ops = { .bc_inv = indy_sc_wback_invalidate }; -void __cpuinit indy_sc_init(void) +void indy_sc_init(void) { if (indy_sc_probe()) { indy_sc_enable(); diff --git a/arch/mips/mm/sc-mips.c b/arch/mips/mm/sc-mips.c index df96da7e939b5..5d01392e3518d 100644 --- a/arch/mips/mm/sc-mips.c +++ b/arch/mips/mm/sc-mips.c @@ -132,7 +132,7 @@ static inline int __init mips_sc_probe(void) return 1; } -int __cpuinit mips_sc_init(void) +int mips_sc_init(void) { int found = mips_sc_probe(); if (found) { diff --git a/arch/mips/mm/sc-r5k.c b/arch/mips/mm/sc-r5k.c index 8bc67720e145f..0216ed6eaa2a9 100644 --- a/arch/mips/mm/sc-r5k.c +++ b/arch/mips/mm/sc-r5k.c @@ -98,7 +98,7 @@ static struct bcache_ops r5k_sc_ops = { .bc_inv = r5k_dma_cache_inv_sc }; -void __cpuinit r5k_sc_init(void) +void r5k_sc_init(void) { if (r5k_sc_probe()) { r5k_sc_enable(); diff --git a/arch/mips/mm/sc-rm7k.c b/arch/mips/mm/sc-rm7k.c index 274af3be1442b..aaffbba337064 100644 --- a/arch/mips/mm/sc-rm7k.c +++ b/arch/mips/mm/sc-rm7k.c @@ -104,7 +104,7 @@ static void blast_rm7k_tcache(void) /* * This function is executed in uncached address space. */ -static __cpuinit void __rm7k_tc_enable(void) +static void __rm7k_tc_enable(void) { int i; @@ -117,7 +117,7 @@ static __cpuinit void __rm7k_tc_enable(void) cache_op(Index_Store_Tag_T, CKSEG0ADDR(i)); } -static __cpuinit void rm7k_tc_enable(void) +static void rm7k_tc_enable(void) { if (read_c0_config() & RM7K_CONF_TE) return; @@ -130,7 +130,7 @@ static __cpuinit void rm7k_tc_enable(void) /* * This function is executed in uncached address space. */ -static __cpuinit void __rm7k_sc_enable(void) +static void __rm7k_sc_enable(void) { int i; @@ -143,7 +143,7 @@ static __cpuinit void __rm7k_sc_enable(void) cache_op(Index_Store_Tag_SD, CKSEG0ADDR(i)); } -static __cpuinit void rm7k_sc_enable(void) +static void rm7k_sc_enable(void) { if (read_c0_config() & RM7K_CONF_SE) return; @@ -184,7 +184,7 @@ static struct bcache_ops rm7k_sc_ops = { * This is a probing function like the one found in c-r4k.c, we look for the * wrap around point with different addresses. */ -static __cpuinit void __probe_tcache(void) +static void __probe_tcache(void) { unsigned long flags, addr, begin, end, pow2; @@ -226,7 +226,7 @@ static __cpuinit void __probe_tcache(void) local_irq_restore(flags); } -void __cpuinit rm7k_sc_init(void) +void rm7k_sc_init(void) { struct cpuinfo_mips *c = ¤t_cpu_data; unsigned int config = read_c0_config(); diff --git a/arch/mips/mm/tlb-r3k.c b/arch/mips/mm/tlb-r3k.c index a63d1ed0827fe..9aca10994cd22 100644 --- a/arch/mips/mm/tlb-r3k.c +++ b/arch/mips/mm/tlb-r3k.c @@ -276,7 +276,7 @@ void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, } } -void __cpuinit tlb_init(void) +void tlb_init(void) { local_flush_tlb_all(); diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c index c643de4c473a8..00b26a67a06d8 100644 --- a/arch/mips/mm/tlb-r4k.c +++ b/arch/mips/mm/tlb-r4k.c @@ -389,7 +389,7 @@ int __init has_transparent_hugepage(void) #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -static int __cpuinitdata ntlb; +static int ntlb; static int __init set_ntlb(char *str) { get_option(&str, &ntlb); @@ -398,7 +398,7 @@ static int __init set_ntlb(char *str) __setup("ntlb=", set_ntlb); -void __cpuinit tlb_init(void) +void tlb_init(void) { /* * You should never change this register: diff --git a/arch/mips/mm/tlb-r8k.c b/arch/mips/mm/tlb-r8k.c index 91c2499f806a2..6a99733a44402 100644 --- a/arch/mips/mm/tlb-r8k.c +++ b/arch/mips/mm/tlb-r8k.c @@ -213,14 +213,14 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) local_irq_restore(flags); } -static void __cpuinit probe_tlb(unsigned long config) +static void probe_tlb(unsigned long config) { struct cpuinfo_mips *c = ¤t_cpu_data; c->tlbsize = 3 * 128; /* 3 sets each 128 entries */ } -void __cpuinit tlb_init(void) +void tlb_init(void) { unsigned int config = read_c0_config(); unsigned long status; diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index a91a7a99f70f3..842b087443a8b 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -136,7 +136,7 @@ static int scratchpad_offset(int i) * why; it's not an issue caused by the core RTL. * */ -static int __cpuinit m4kc_tlbp_war(void) +static int m4kc_tlbp_war(void) { return (current_cpu_data.processor_id & 0xffff00) == (PRID_COMP_MIPS | PRID_IMP_4KC); @@ -181,9 +181,9 @@ UASM_L_LA(_large_segbits_fault) UASM_L_LA(_tlb_huge_update) #endif -static int __cpuinitdata hazard_instance; +static int hazard_instance; -static void __cpuinit uasm_bgezl_hazard(u32 **p, +static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance) { @@ -196,7 +196,7 @@ static void __cpuinit uasm_bgezl_hazard(u32 **p, } } -static void __cpuinit uasm_bgezl_label(struct uasm_label **l, +static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance) { @@ -305,7 +305,7 @@ static int check_for_high_segbits __cpuinitdata; static unsigned int kscratch_used_mask __cpuinitdata; -static int __cpuinit allocate_kscratch(void) +static int allocate_kscratch(void) { int r; unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask; @@ -326,7 +326,7 @@ static int scratch_reg __cpuinitdata; static int pgd_reg __cpuinitdata; enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch}; -static struct work_registers __cpuinit build_get_work_registers(u32 **p) +static struct work_registers build_get_work_registers(u32 **p) { struct work_registers r; @@ -382,7 +382,7 @@ static struct work_registers __cpuinit build_get_work_registers(u32 **p) return r; } -static void __cpuinit build_restore_work_registers(u32 **p) +static void build_restore_work_registers(u32 **p) { if (scratch_reg > 0) { UASM_i_MFC0(p, 1, 31, scratch_reg); @@ -407,7 +407,7 @@ extern unsigned long pgd_current[]; /* * The R3000 TLB handler is simple. */ -static void __cpuinit build_r3000_tlb_refill_handler(void) +static void build_r3000_tlb_refill_handler(void) { long pgdc = (long)pgd_current; u32 *p; @@ -476,7 +476,7 @@ static u32 final_handler[64] __cpuinitdata; * * As if we MIPS hackers wouldn't know how to nop pipelines happy ... */ -static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p) +static void __maybe_unused build_tlb_probe_entry(u32 **p) { switch (current_cpu_type()) { /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ @@ -500,7 +500,7 @@ static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p) */ enum tlb_write_entry { tlb_random, tlb_indexed }; -static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, +static void build_tlb_write_entry(u32 **p, struct uasm_label **l, struct uasm_reloc **r, enum tlb_write_entry wmode) { @@ -636,7 +636,7 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, } } -static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p, +static __maybe_unused void build_convert_pte_to_entrylo(u32 **p, unsigned int reg) { if (cpu_has_rixi) { @@ -652,7 +652,7 @@ static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p, #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT -static __cpuinit void build_restore_pagemask(u32 **p, +static void build_restore_pagemask(u32 **p, struct uasm_reloc **r, unsigned int tmp, enum label_id lid, @@ -695,7 +695,7 @@ static __cpuinit void build_restore_pagemask(u32 **p, } } -static __cpuinit void build_huge_tlb_write_entry(u32 **p, +static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int tmp, @@ -715,7 +715,7 @@ static __cpuinit void build_huge_tlb_write_entry(u32 **p, /* * Check if Huge PTE is present, if so then jump to LABEL. */ -static void __cpuinit +static void build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, unsigned int pmd, int lid) { @@ -728,7 +728,7 @@ build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, } } -static __cpuinit void build_huge_update_entries(u32 **p, +static void build_huge_update_entries(u32 **p, unsigned int pte, unsigned int tmp) { @@ -760,7 +760,7 @@ static __cpuinit void build_huge_update_entries(u32 **p, UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ } -static __cpuinit void build_huge_handler_tail(u32 **p, +static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r, struct uasm_label **l, unsigned int pte, @@ -783,7 +783,7 @@ static __cpuinit void build_huge_handler_tail(u32 **p, * TMP and PTR are scratch. * TMP will be clobbered, PTR will hold the pmd entry. */ -static void __cpuinit +static void build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int tmp, unsigned int ptr) { @@ -875,7 +875,7 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, * BVADDR is the faulting address, PTR is scratch. * PTR will hold the pgd for vmalloc. */ -static void __cpuinit +static void build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int bvaddr, unsigned int ptr, enum vmalloc64_mode mode) @@ -945,7 +945,7 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, * TMP and PTR are scratch. * TMP will be clobbered, PTR will hold the pgd entry. */ -static void __cpuinit __maybe_unused +static void __maybe_unused build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) { long pgdc = (long)pgd_current; @@ -980,7 +980,7 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) #endif /* !CONFIG_64BIT */ -static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx) +static void build_adjust_context(u32 **p, unsigned int ctx) { unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); @@ -1006,7 +1006,7 @@ static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx) uasm_i_andi(p, ctx, ctx, mask); } -static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) +static void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) { /* * Bug workaround for the Nevada. It seems as if under certain @@ -1031,7 +1031,7 @@ static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ } -static void __cpuinit build_update_entries(u32 **p, unsigned int tmp, +static void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep) { /* @@ -1094,7 +1094,7 @@ struct mips_huge_tlb_info { bool need_reload_pte; }; -static struct mips_huge_tlb_info __cpuinit +static struct mips_huge_tlb_info build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int tmp, unsigned int ptr, int c0_scratch) @@ -1273,7 +1273,7 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, */ #define MIPS64_REFILL_INSNS 32 -static void __cpuinit build_r4000_tlb_refill_handler(void) +static void build_r4000_tlb_refill_handler(void) { u32 *p = tlb_handler; struct uasm_label *l = labels; @@ -1461,7 +1461,7 @@ u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned; #ifdef CONFIG_MIPS_PGD_C0_CONTEXT u32 tlbmiss_handler_setup_pgd_array[16] __cacheline_aligned; -static void __cpuinit build_r4000_setup_pgd(void) +static void build_r4000_setup_pgd(void) { const int a0 = 4; const int a1 = 5; @@ -1509,7 +1509,7 @@ static void __cpuinit build_r4000_setup_pgd(void) } #endif -static void __cpuinit +static void iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) { #ifdef CONFIG_SMP @@ -1529,7 +1529,7 @@ iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) #endif } -static void __cpuinit +static void iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, unsigned int mode) { @@ -1589,7 +1589,7 @@ iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, * the page table where this PTE is located, PTE will be re-loaded * with it's original value. */ -static void __cpuinit +static void build_pte_present(u32 **p, struct uasm_reloc **r, int pte, int ptr, int scratch, enum label_id lid) { @@ -1617,7 +1617,7 @@ build_pte_present(u32 **p, struct uasm_reloc **r, } /* Make PTE valid, store result in PTR. */ -static void __cpuinit +static void build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr) { @@ -1630,7 +1630,7 @@ build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, * Check if PTE can be written to, if not branch to LABEL. Regardless * restore PTE with value from PTR when done. */ -static void __cpuinit +static void build_pte_writable(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, int scratch, enum label_id lid) @@ -1650,7 +1650,7 @@ build_pte_writable(u32 **p, struct uasm_reloc **r, /* Make PTE writable, update software status bits as well, then store * at PTR. */ -static void __cpuinit +static void build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr) { @@ -1664,7 +1664,7 @@ build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, * Check if PTE can be modified, if not branch to LABEL. Regardless * restore PTE with value from PTR when done. */ -static void __cpuinit +static void build_pte_modifiable(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, int scratch, enum label_id lid) @@ -1693,7 +1693,7 @@ build_pte_modifiable(u32 **p, struct uasm_reloc **r, * This places the pte into ENTRYLO0 and writes it with tlbwi. * Then it returns. */ -static void __cpuinit +static void build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) { uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ @@ -1709,7 +1709,7 @@ build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) * may have the probe fail bit set as a result of a trap on a * kseg2 access, i.e. without refill. Then it returns. */ -static void __cpuinit +static void build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int pte, unsigned int tmp) @@ -1727,7 +1727,7 @@ build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, uasm_i_rfe(p); /* branch delay */ } -static void __cpuinit +static void build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, unsigned int ptr) { @@ -1747,7 +1747,7 @@ build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, uasm_i_tlbp(p); /* load delay */ } -static void __cpuinit build_r3000_tlb_load_handler(void) +static void build_r3000_tlb_load_handler(void) { u32 *p = handle_tlbl; struct uasm_label *l = labels; @@ -1777,7 +1777,7 @@ static void __cpuinit build_r3000_tlb_load_handler(void) dump_handler("r3000_tlb_load", handle_tlbl, ARRAY_SIZE(handle_tlbl)); } -static void __cpuinit build_r3000_tlb_store_handler(void) +static void build_r3000_tlb_store_handler(void) { u32 *p = handle_tlbs; struct uasm_label *l = labels; @@ -1807,7 +1807,7 @@ static void __cpuinit build_r3000_tlb_store_handler(void) dump_handler("r3000_tlb_store", handle_tlbs, ARRAY_SIZE(handle_tlbs)); } -static void __cpuinit build_r3000_tlb_modify_handler(void) +static void build_r3000_tlb_modify_handler(void) { u32 *p = handle_tlbm; struct uasm_label *l = labels; @@ -1841,7 +1841,7 @@ static void __cpuinit build_r3000_tlb_modify_handler(void) /* * R4000 style TLB load/store/modify handlers. */ -static struct work_registers __cpuinit +static struct work_registers build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, struct uasm_reloc **r) { @@ -1877,7 +1877,7 @@ build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, return wr; } -static void __cpuinit +static void build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int tmp, unsigned int ptr) @@ -1895,7 +1895,7 @@ build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, #endif } -static void __cpuinit build_r4000_tlb_load_handler(void) +static void build_r4000_tlb_load_handler(void) { u32 *p = handle_tlbl; struct uasm_label *l = labels; @@ -2051,7 +2051,7 @@ static void __cpuinit build_r4000_tlb_load_handler(void) dump_handler("r4000_tlb_load", handle_tlbl, ARRAY_SIZE(handle_tlbl)); } -static void __cpuinit build_r4000_tlb_store_handler(void) +static void build_r4000_tlb_store_handler(void) { u32 *p = handle_tlbs; struct uasm_label *l = labels; @@ -2105,7 +2105,7 @@ static void __cpuinit build_r4000_tlb_store_handler(void) dump_handler("r4000_tlb_store", handle_tlbs, ARRAY_SIZE(handle_tlbs)); } -static void __cpuinit build_r4000_tlb_modify_handler(void) +static void build_r4000_tlb_modify_handler(void) { u32 *p = handle_tlbm; struct uasm_label *l = labels; @@ -2160,7 +2160,7 @@ static void __cpuinit build_r4000_tlb_modify_handler(void) dump_handler("r4000_tlb_modify", handle_tlbm, ARRAY_SIZE(handle_tlbm)); } -void __cpuinit build_tlb_refill_handler(void) +void build_tlb_refill_handler(void) { /* * The refill handler is generated per-CPU, multi-node systems @@ -2226,7 +2226,7 @@ void __cpuinit build_tlb_refill_handler(void) } } -void __cpuinit flush_tlb_handlers(void) +void flush_tlb_handlers(void) { local_flush_icache_range((unsigned long)handle_tlbl, (unsigned long)handle_tlbl + sizeof(handle_tlbl)); diff --git a/arch/mips/mti-malta/malta-smtc.c b/arch/mips/mti-malta/malta-smtc.c index becbf47506a59..c4849904f0139 100644 --- a/arch/mips/mti-malta/malta-smtc.c +++ b/arch/mips/mti-malta/malta-smtc.c @@ -32,7 +32,7 @@ static void msmtc_send_ipi_mask(const struct cpumask *mask, unsigned int action) /* * Post-config but pre-boot cleanup entry point */ -static void __cpuinit msmtc_init_secondary(void) +static void msmtc_init_secondary(void) { int myvpe; @@ -53,7 +53,7 @@ static void __cpuinit msmtc_init_secondary(void) /* * Platform "CPU" startup hook */ -static void __cpuinit msmtc_boot_secondary(int cpu, struct task_struct *idle) +static void msmtc_boot_secondary(int cpu, struct task_struct *idle) { smtc_boot_secondary(cpu, idle); } @@ -61,7 +61,7 @@ static void __cpuinit msmtc_boot_secondary(int cpu, struct task_struct *idle) /* * SMP initialization finalization entry point */ -static void __cpuinit msmtc_smp_finish(void) +static void msmtc_smp_finish(void) { smtc_smp_finish(); } diff --git a/arch/mips/mti-malta/malta-time.c b/arch/mips/mti-malta/malta-time.c index 0ad305f75802b..53aad4a353756 100644 --- a/arch/mips/mti-malta/malta-time.c +++ b/arch/mips/mti-malta/malta-time.c @@ -150,7 +150,7 @@ static void __init plat_perf_setup(void) } } -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { #ifdef MSC01E_INT_BASE if (cpu_has_veic) { diff --git a/arch/mips/mti-sead3/sead3-time.c b/arch/mips/mti-sead3/sead3-time.c index 96b42eb9b5e22..a43ea3cc0a3bf 100644 --- a/arch/mips/mti-sead3/sead3-time.c +++ b/arch/mips/mti-sead3/sead3-time.c @@ -91,7 +91,7 @@ static void __init plat_perf_setup(void) } } -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { if (cpu_has_vint) set_vi_handler(cp0_compare_irq, mips_timer_dispatch); diff --git a/arch/mips/netlogic/common/smp.c b/arch/mips/netlogic/common/smp.c index ffba52489bef7..c01d554f64e14 100644 --- a/arch/mips/netlogic/common/smp.c +++ b/arch/mips/netlogic/common/smp.c @@ -116,7 +116,7 @@ void nlm_early_init_secondary(int cpu) /* * Code to run on secondary just after probing the CPU */ -static void __cpuinit nlm_init_secondary(void) +static void nlm_init_secondary(void) { int hwtid; @@ -252,7 +252,7 @@ static int nlm_parse_cpumask(cpumask_t *wakeup_mask) return 0; } -int __cpuinit nlm_wakeup_secondary_cpus(void) +int nlm_wakeup_secondary_cpus(void) { unsigned long reset_vec; char *reset_data; diff --git a/arch/mips/netlogic/common/time.c b/arch/mips/netlogic/common/time.c index 5c56555380bb1..045a396c57ced 100644 --- a/arch/mips/netlogic/common/time.c +++ b/arch/mips/netlogic/common/time.c @@ -54,7 +54,7 @@ #error "Unknown CPU" #endif -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { return IRQ_TIMER; } diff --git a/arch/mips/netlogic/xlr/wakeup.c b/arch/mips/netlogic/xlr/wakeup.c index 3ebf7411d67b1..37a40927ed719 100644 --- a/arch/mips/netlogic/xlr/wakeup.c +++ b/arch/mips/netlogic/xlr/wakeup.c @@ -49,7 +49,7 @@ #include #include -int __cpuinit xlr_wakeup_secondary_cpus(void) +int xlr_wakeup_secondary_cpus(void) { struct nlm_soc_info *nodep; unsigned int i, j, boot_cpu; diff --git a/arch/mips/pci/pci-ip27.c b/arch/mips/pci/pci-ip27.c index 6eb65e44d9e4a..2946bdf654389 100644 --- a/arch/mips/pci/pci-ip27.c +++ b/arch/mips/pci/pci-ip27.c @@ -42,7 +42,7 @@ int irq_to_slot[MAX_PCI_BUSSES * MAX_DEVICES_PER_PCIBUS]; extern struct pci_ops bridge_pci_ops; -int __cpuinit bridge_probe(nasid_t nasid, int widget_id, int masterwid) +int bridge_probe(nasid_t nasid, int widget_id, int masterwid) { unsigned long offset = NODE_OFFSET(nasid); struct bridge_controller *bc; diff --git a/arch/mips/pmcs-msp71xx/msp_smtc.c b/arch/mips/pmcs-msp71xx/msp_smtc.c index c8dcc1c01e181..6aa91401ddfc3 100644 --- a/arch/mips/pmcs-msp71xx/msp_smtc.c +++ b/arch/mips/pmcs-msp71xx/msp_smtc.c @@ -33,7 +33,7 @@ static void msp_smtc_send_ipi_mask(const struct cpumask *mask, /* * Post-config but pre-boot cleanup entry point */ -static void __cpuinit msp_smtc_init_secondary(void) +static void msp_smtc_init_secondary(void) { int myvpe; @@ -48,7 +48,7 @@ static void __cpuinit msp_smtc_init_secondary(void) /* * Platform "CPU" startup hook */ -static void __cpuinit msp_smtc_boot_secondary(int cpu, +static void msp_smtc_boot_secondary(int cpu, struct task_struct *idle) { smtc_boot_secondary(cpu, idle); @@ -57,7 +57,7 @@ static void __cpuinit msp_smtc_boot_secondary(int cpu, /* * SMP initialization finalization entry point */ -static void __cpuinit msp_smtc_smp_finish(void) +static void msp_smtc_smp_finish(void) { smtc_smp_finish(); } diff --git a/arch/mips/pmcs-msp71xx/msp_time.c b/arch/mips/pmcs-msp71xx/msp_time.c index 8f12ecc55ace5..fea917be0ff10 100644 --- a/arch/mips/pmcs-msp71xx/msp_time.c +++ b/arch/mips/pmcs-msp71xx/msp_time.c @@ -88,7 +88,7 @@ void __init plat_time_init(void) mips_hpt_frequency = cpu_rate/2; } -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { /* MIPS_MT modes may want timer for second VPE */ if ((get_current_vpe()) && !tim_installed) { diff --git a/arch/mips/pnx833x/common/interrupts.c b/arch/mips/pnx833x/common/interrupts.c index a4a90596c0ad4..e460865873c1c 100644 --- a/arch/mips/pnx833x/common/interrupts.c +++ b/arch/mips/pnx833x/common/interrupts.c @@ -281,7 +281,7 @@ void __init arch_init_irq(void) write_c0_status(read_c0_status() | IE_IRQ2); } -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { if (cpu_has_vint) set_vi_handler(cp0_compare_irq, pnx833x_timer_dispatch); diff --git a/arch/mips/powertv/time.c b/arch/mips/powertv/time.c index 9fd7b67f2af7c..f38b0d45eca90 100644 --- a/arch/mips/powertv/time.c +++ b/arch/mips/powertv/time.c @@ -25,7 +25,7 @@ #include "powertv-clock.h" -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { return irq_mips_timer; } diff --git a/arch/mips/ralink/irq.c b/arch/mips/ralink/irq.c index 320b1f1043fff..781b3d14a489e 100644 --- a/arch/mips/ralink/irq.c +++ b/arch/mips/ralink/irq.c @@ -73,7 +73,7 @@ static struct irq_chip ralink_intc_irq_chip = { .irq_mask_ack = ralink_intc_irq_mask, }; -unsigned int __cpuinit get_c0_compare_int(void) +unsigned int get_c0_compare_int(void) { return CP0_LEGACY_COMPARE_IRQ; } diff --git a/arch/mips/sgi-ip27/ip27-init.c b/arch/mips/sgi-ip27/ip27-init.c index d41b1c6fb0329..ee736bd103f85 100644 --- a/arch/mips/sgi-ip27/ip27-init.c +++ b/arch/mips/sgi-ip27/ip27-init.c @@ -54,7 +54,7 @@ extern void pcibr_setup(cnodeid_t); extern void xtalk_probe_node(cnodeid_t nid); -static void __cpuinit per_hub_init(cnodeid_t cnode) +static void per_hub_init(cnodeid_t cnode) { struct hub_data *hub = hub_data(cnode); nasid_t nasid = COMPACT_TO_NASID_NODEID(cnode); @@ -110,7 +110,7 @@ static void __cpuinit per_hub_init(cnodeid_t cnode) } } -void __cpuinit per_cpu_init(void) +void per_cpu_init(void) { int cpu = smp_processor_id(); int slice = LOCAL_HUB_L(PI_CPU_NUM); diff --git a/arch/mips/sgi-ip27/ip27-smp.c b/arch/mips/sgi-ip27/ip27-smp.c index f94638141b20e..f4ea8aa79ba2d 100644 --- a/arch/mips/sgi-ip27/ip27-smp.c +++ b/arch/mips/sgi-ip27/ip27-smp.c @@ -173,12 +173,12 @@ static void ip27_send_ipi_mask(const struct cpumask *mask, unsigned int action) ip27_send_ipi_single(i, action); } -static void __cpuinit ip27_init_secondary(void) +static void ip27_init_secondary(void) { per_cpu_init(); } -static void __cpuinit ip27_smp_finish(void) +static void ip27_smp_finish(void) { extern void hub_rt_clock_event_init(void); @@ -195,7 +195,7 @@ static void __init ip27_cpus_done(void) * set sp to the kernel stack of the newly created idle process, gp to the proc * struct so that current_thread_info() will work. */ -static void __cpuinit ip27_boot_secondary(int cpu, struct task_struct *idle) +static void ip27_boot_secondary(int cpu, struct task_struct *idle) { unsigned long gp = (unsigned long)task_thread_info(idle); unsigned long sp = __KSTK_TOS(idle); diff --git a/arch/mips/sgi-ip27/ip27-timer.c b/arch/mips/sgi-ip27/ip27-timer.c index 2e21b761cb9c7..1d97eaba0c5f0 100644 --- a/arch/mips/sgi-ip27/ip27-timer.c +++ b/arch/mips/sgi-ip27/ip27-timer.c @@ -106,7 +106,7 @@ struct irqaction hub_rt_irqaction = { #define NSEC_PER_CYCLE 800 #define CYCLES_PER_SEC (NSEC_PER_SEC / NSEC_PER_CYCLE) -void __cpuinit hub_rt_clock_event_init(void) +void hub_rt_clock_event_init(void) { unsigned int cpu = smp_processor_id(); struct clock_event_device *cd = &per_cpu(hub_rt_clockevent, cpu); @@ -173,7 +173,7 @@ void __init plat_time_init(void) hub_rt_clock_event_init(); } -void __cpuinit cpu_time_init(void) +void cpu_time_init(void) { lboard_t *board; klcpu_t *cpu; @@ -194,7 +194,7 @@ void __cpuinit cpu_time_init(void) set_c0_status(SRB_TIMOCLK); } -void __cpuinit hub_rtc_init(cnodeid_t cnode) +void hub_rtc_init(cnodeid_t cnode) { /* diff --git a/arch/mips/sgi-ip27/ip27-xtalk.c b/arch/mips/sgi-ip27/ip27-xtalk.c index a4df7d0f6f12d..d59b820f528d1 100644 --- a/arch/mips/sgi-ip27/ip27-xtalk.c +++ b/arch/mips/sgi-ip27/ip27-xtalk.c @@ -23,7 +23,7 @@ extern int bridge_probe(nasid_t nasid, int widget, int masterwid); -static int __cpuinit probe_one_port(nasid_t nasid, int widget, int masterwid) +static int probe_one_port(nasid_t nasid, int widget, int masterwid) { widgetreg_t widget_id; xwidget_part_num_t partnum; @@ -47,7 +47,7 @@ static int __cpuinit probe_one_port(nasid_t nasid, int widget, int masterwid) return 0; } -static int __cpuinit xbow_probe(nasid_t nasid) +static int xbow_probe(nasid_t nasid) { lboard_t *brd; klxbow_t *xbow_p; @@ -100,7 +100,7 @@ static int __cpuinit xbow_probe(nasid_t nasid) return 0; } -void __cpuinit xtalk_probe_node(cnodeid_t nid) +void xtalk_probe_node(cnodeid_t nid) { volatile u64 hubreg; nasid_t nasid; diff --git a/arch/mips/sibyte/bcm1480/smp.c b/arch/mips/sibyte/bcm1480/smp.c index de88e22694a05..54e2c4de15c16 100644 --- a/arch/mips/sibyte/bcm1480/smp.c +++ b/arch/mips/sibyte/bcm1480/smp.c @@ -60,7 +60,7 @@ static void *mailbox_0_regs[] = { /* * SMP init and finish on secondary CPUs */ -void __cpuinit bcm1480_smp_init(void) +void bcm1480_smp_init(void) { unsigned int imask = STATUSF_IP4 | STATUSF_IP3 | STATUSF_IP2 | STATUSF_IP1 | STATUSF_IP0; @@ -95,7 +95,7 @@ static void bcm1480_send_ipi_mask(const struct cpumask *mask, /* * Code to run on secondary just after probing the CPU */ -static void __cpuinit bcm1480_init_secondary(void) +static void bcm1480_init_secondary(void) { extern void bcm1480_smp_init(void); @@ -106,7 +106,7 @@ static void __cpuinit bcm1480_init_secondary(void) * Do any tidying up before marking online and running the idle * loop */ -static void __cpuinit bcm1480_smp_finish(void) +static void bcm1480_smp_finish(void) { extern void sb1480_clockevent_init(void); @@ -125,7 +125,7 @@ static void bcm1480_cpus_done(void) * Setup the PC, SP, and GP of a secondary processor and start it * running! */ -static void __cpuinit bcm1480_boot_secondary(int cpu, struct task_struct *idle) +static void bcm1480_boot_secondary(int cpu, struct task_struct *idle) { int retval; diff --git a/arch/mips/sibyte/sb1250/smp.c b/arch/mips/sibyte/sb1250/smp.c index 285cfef4ebc08..d7b942db0ea51 100644 --- a/arch/mips/sibyte/sb1250/smp.c +++ b/arch/mips/sibyte/sb1250/smp.c @@ -48,7 +48,7 @@ static void *mailbox_regs[] = { /* * SMP init and finish on secondary CPUs */ -void __cpuinit sb1250_smp_init(void) +void sb1250_smp_init(void) { unsigned int imask = STATUSF_IP4 | STATUSF_IP3 | STATUSF_IP2 | STATUSF_IP1 | STATUSF_IP0; @@ -83,7 +83,7 @@ static inline void sb1250_send_ipi_mask(const struct cpumask *mask, /* * Code to run on secondary just after probing the CPU */ -static void __cpuinit sb1250_init_secondary(void) +static void sb1250_init_secondary(void) { extern void sb1250_smp_init(void); @@ -94,7 +94,7 @@ static void __cpuinit sb1250_init_secondary(void) * Do any tidying up before marking online and running the idle * loop */ -static void __cpuinit sb1250_smp_finish(void) +static void sb1250_smp_finish(void) { extern void sb1250_clockevent_init(void); @@ -113,7 +113,7 @@ static void sb1250_cpus_done(void) * Setup the PC, SP, and GP of a secondary processor and start it * running! */ -static void __cpuinit sb1250_boot_secondary(int cpu, struct task_struct *idle) +static void sb1250_boot_secondary(int cpu, struct task_struct *idle) { int retval; diff --git a/arch/openrisc/kernel/setup.c b/arch/openrisc/kernel/setup.c index f4d5bedc3b4f7..d7359ffbcbdd3 100644 --- a/arch/openrisc/kernel/setup.c +++ b/arch/openrisc/kernel/setup.c @@ -267,7 +267,7 @@ void __init detect_unit_config(unsigned long upr, unsigned long mask, * */ -void __cpuinit calibrate_delay(void) +void calibrate_delay(void) { const int *val; struct device_node *cpu = NULL; diff --git a/arch/parisc/kernel/firmware.c b/arch/parisc/kernel/firmware.c index f65fa480c905f..4e6a60470bce8 100644 --- a/arch/parisc/kernel/firmware.c +++ b/arch/parisc/kernel/firmware.c @@ -150,7 +150,7 @@ static void convert_to_wide(unsigned long *addr) } #ifdef CONFIG_64BIT -void __cpuinit set_firmware_width_unlocked(void) +void set_firmware_width_unlocked(void) { int ret; @@ -167,7 +167,7 @@ void __cpuinit set_firmware_width_unlocked(void) * This function must be called before any pdc_* function that uses the * convert_to_wide function. */ -void __cpuinit set_firmware_width(void) +void set_firmware_width(void) { unsigned long flags; spin_lock_irqsave(&pdc_lock, flags); @@ -175,11 +175,11 @@ void __cpuinit set_firmware_width(void) spin_unlock_irqrestore(&pdc_lock, flags); } #else -void __cpuinit set_firmware_width_unlocked(void) { +void set_firmware_width_unlocked(void) { return; } -void __cpuinit set_firmware_width(void) { +void set_firmware_width(void) { return; } #endif /*CONFIG_64BIT*/ @@ -301,7 +301,7 @@ int pdc_chassis_warn(unsigned long *warn) return retval; } -int __cpuinit pdc_coproc_cfg_unlocked(struct pdc_coproc_cfg *pdc_coproc_info) +int pdc_coproc_cfg_unlocked(struct pdc_coproc_cfg *pdc_coproc_info) { int ret; @@ -322,7 +322,7 @@ int __cpuinit pdc_coproc_cfg_unlocked(struct pdc_coproc_cfg *pdc_coproc_info) * This PDC call returns the presence and status of all the coprocessors * attached to the processor. */ -int __cpuinit pdc_coproc_cfg(struct pdc_coproc_cfg *pdc_coproc_info) +int pdc_coproc_cfg(struct pdc_coproc_cfg *pdc_coproc_info) { int ret; unsigned long flags; diff --git a/arch/parisc/kernel/hardware.c b/arch/parisc/kernel/hardware.c index c22c3d84e28b5..5b2c5f3940358 100644 --- a/arch/parisc/kernel/hardware.c +++ b/arch/parisc/kernel/hardware.c @@ -1368,7 +1368,7 @@ const char *parisc_hardware_description(struct parisc_device_id *id) /* Interpret hversion (ret[0]) from PDC_MODEL(4)/PDC_MODEL_INFO(0) */ -enum cpu_type __cpuinit +enum cpu_type parisc_get_cpu_type(unsigned long hversion) { struct hp_cpu_type_mask *ptr; diff --git a/arch/parisc/kernel/processor.c b/arch/parisc/kernel/processor.c index c8fb61ed32f4e..a128a94204e68 100644 --- a/arch/parisc/kernel/processor.c +++ b/arch/parisc/kernel/processor.c @@ -73,7 +73,7 @@ extern int update_cr16_clocksource(void); /* from time.c */ * * FIXME: doesn't do much yet... */ -static void __cpuinit +static void init_percpu_prof(unsigned long cpunum) { struct cpuinfo_parisc *p; @@ -92,7 +92,7 @@ init_percpu_prof(unsigned long cpunum) * (return 1). If so, initialize the chip and tell other partners in crime * they have work to do. */ -static int __cpuinit processor_probe(struct parisc_device *dev) +static int processor_probe(struct parisc_device *dev) { unsigned long txn_addr; unsigned long cpuid; @@ -299,7 +299,7 @@ void __init collect_boot_cpu_data(void) * * o Enable CPU profiling hooks. */ -int __cpuinit init_per_cpu(int cpunum) +int init_per_cpu(int cpunum) { int ret; struct pdc_coproc_cfg coproc_cfg; diff --git a/arch/parisc/kernel/smp.c b/arch/parisc/kernel/smp.c index e3614fb343e5e..71cb54ab9eaa5 100644 --- a/arch/parisc/kernel/smp.c +++ b/arch/parisc/kernel/smp.c @@ -64,7 +64,7 @@ volatile struct task_struct *smp_init_current_idle_task; /* track which CPU is booting */ static volatile int cpu_now_booting __cpuinitdata; -static int parisc_max_cpus __cpuinitdata = 1; +static int parisc_max_cpus = 1; static DEFINE_PER_CPU(spinlock_t, ipi_lock); @@ -328,7 +328,7 @@ void __init smp_callin(void) /* * Bring one cpu online. */ -int __cpuinit smp_boot_one_cpu(int cpuid, struct task_struct *idle) +int smp_boot_one_cpu(int cpuid, struct task_struct *idle) { const struct cpuinfo_parisc *p = &per_cpu(cpu_data, cpuid); long timeout; @@ -424,7 +424,7 @@ void smp_cpus_done(unsigned int cpu_max) } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { if (cpu != 0 && cpu < parisc_max_cpus) smp_boot_one_cpu(cpu, tidle); diff --git a/arch/powerpc/include/asm/rtas.h b/arch/powerpc/include/asm/rtas.h index c5d5cb36f6c66..50611b6438a86 100644 --- a/arch/powerpc/include/asm/rtas.h +++ b/arch/powerpc/include/asm/rtas.h @@ -351,8 +351,8 @@ static inline u32 rtas_config_addr(int busno, int devfn, int reg) (devfn << 8) | (reg & 0xff); } -extern void __cpuinit rtas_give_timebase(void); -extern void __cpuinit rtas_take_timebase(void); +extern void rtas_give_timebase(void); +extern void rtas_take_timebase(void); #ifdef CONFIG_PPC_RTAS static inline int page_is_rtas_user_buf(unsigned long pfn) diff --git a/arch/powerpc/include/asm/vdso.h b/arch/powerpc/include/asm/vdso.h index 50f261bc3e959..0d9cecddf8a4f 100644 --- a/arch/powerpc/include/asm/vdso.h +++ b/arch/powerpc/include/asm/vdso.h @@ -22,7 +22,7 @@ extern unsigned long vdso64_rt_sigtramp; extern unsigned long vdso32_sigtramp; extern unsigned long vdso32_rt_sigtramp; -int __cpuinit vdso_getcpu_init(void); +int vdso_getcpu_init(void); #else /* __ASSEMBLY__ */ diff --git a/arch/powerpc/kernel/cacheinfo.c b/arch/powerpc/kernel/cacheinfo.c index b4437e8a7a8f6..03553af1334b1 100644 --- a/arch/powerpc/kernel/cacheinfo.c +++ b/arch/powerpc/kernel/cacheinfo.c @@ -131,7 +131,7 @@ static const char *cache_type_string(const struct cache *cache) return cache_type_info[cache->type].name; } -static void __cpuinit cache_init(struct cache *cache, int type, int level, struct device_node *ofnode) +static void cache_init(struct cache *cache, int type, int level, struct device_node *ofnode) { cache->type = type; cache->level = level; @@ -140,7 +140,7 @@ static void __cpuinit cache_init(struct cache *cache, int type, int level, struc list_add(&cache->list, &cache_list); } -static struct cache *__cpuinit new_cache(int type, int level, struct device_node *ofnode) +static struct cache *new_cache(int type, int level, struct device_node *ofnode) { struct cache *cache; @@ -324,7 +324,7 @@ static bool cache_node_is_unified(const struct device_node *np) return of_get_property(np, "cache-unified", NULL); } -static struct cache *__cpuinit cache_do_one_devnode_unified(struct device_node *node, int level) +static struct cache *cache_do_one_devnode_unified(struct device_node *node, int level) { struct cache *cache; @@ -335,7 +335,7 @@ static struct cache *__cpuinit cache_do_one_devnode_unified(struct device_node * return cache; } -static struct cache *__cpuinit cache_do_one_devnode_split(struct device_node *node, int level) +static struct cache *cache_do_one_devnode_split(struct device_node *node, int level) { struct cache *dcache, *icache; @@ -357,7 +357,7 @@ static struct cache *__cpuinit cache_do_one_devnode_split(struct device_node *no return NULL; } -static struct cache *__cpuinit cache_do_one_devnode(struct device_node *node, int level) +static struct cache *cache_do_one_devnode(struct device_node *node, int level) { struct cache *cache; @@ -369,7 +369,7 @@ static struct cache *__cpuinit cache_do_one_devnode(struct device_node *node, in return cache; } -static struct cache *__cpuinit cache_lookup_or_instantiate(struct device_node *node, int level) +static struct cache *cache_lookup_or_instantiate(struct device_node *node, int level) { struct cache *cache; @@ -385,7 +385,7 @@ static struct cache *__cpuinit cache_lookup_or_instantiate(struct device_node *n return cache; } -static void __cpuinit link_cache_lists(struct cache *smaller, struct cache *bigger) +static void link_cache_lists(struct cache *smaller, struct cache *bigger) { while (smaller->next_local) { if (smaller->next_local == bigger) @@ -396,13 +396,13 @@ static void __cpuinit link_cache_lists(struct cache *smaller, struct cache *bigg smaller->next_local = bigger; } -static void __cpuinit do_subsidiary_caches_debugcheck(struct cache *cache) +static void do_subsidiary_caches_debugcheck(struct cache *cache) { WARN_ON_ONCE(cache->level != 1); WARN_ON_ONCE(strcmp(cache->ofnode->type, "cpu")); } -static void __cpuinit do_subsidiary_caches(struct cache *cache) +static void do_subsidiary_caches(struct cache *cache) { struct device_node *subcache_node; int level = cache->level; @@ -423,7 +423,7 @@ static void __cpuinit do_subsidiary_caches(struct cache *cache) } } -static struct cache *__cpuinit cache_chain_instantiate(unsigned int cpu_id) +static struct cache *cache_chain_instantiate(unsigned int cpu_id) { struct device_node *cpu_node; struct cache *cpu_cache = NULL; @@ -448,7 +448,7 @@ static struct cache *__cpuinit cache_chain_instantiate(unsigned int cpu_id) return cpu_cache; } -static struct cache_dir *__cpuinit cacheinfo_create_cache_dir(unsigned int cpu_id) +static struct cache_dir *cacheinfo_create_cache_dir(unsigned int cpu_id) { struct cache_dir *cache_dir; struct device *dev; @@ -653,7 +653,7 @@ static struct kobj_type cache_index_type = { .default_attrs = cache_index_default_attrs, }; -static void __cpuinit cacheinfo_create_index_opt_attrs(struct cache_index_dir *dir) +static void cacheinfo_create_index_opt_attrs(struct cache_index_dir *dir) { const char *cache_name; const char *cache_type; @@ -696,7 +696,7 @@ static void __cpuinit cacheinfo_create_index_opt_attrs(struct cache_index_dir *d kfree(buf); } -static void __cpuinit cacheinfo_create_index_dir(struct cache *cache, int index, struct cache_dir *cache_dir) +static void cacheinfo_create_index_dir(struct cache *cache, int index, struct cache_dir *cache_dir) { struct cache_index_dir *index_dir; int rc; @@ -722,7 +722,7 @@ static void __cpuinit cacheinfo_create_index_dir(struct cache *cache, int index, kfree(index_dir); } -static void __cpuinit cacheinfo_sysfs_populate(unsigned int cpu_id, struct cache *cache_list) +static void cacheinfo_sysfs_populate(unsigned int cpu_id, struct cache *cache_list) { struct cache_dir *cache_dir; struct cache *cache; @@ -740,7 +740,7 @@ static void __cpuinit cacheinfo_sysfs_populate(unsigned int cpu_id, struct cache } } -void __cpuinit cacheinfo_cpu_online(unsigned int cpu_id) +void cacheinfo_cpu_online(unsigned int cpu_id) { struct cache *cache; diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c index 19f49f5b535f1..6e87f4ec2ce74 100644 --- a/arch/powerpc/kernel/rtas.c +++ b/arch/powerpc/kernel/rtas.c @@ -1192,7 +1192,7 @@ int __init early_init_dt_scan_rtas(unsigned long node, static arch_spinlock_t timebase_lock; static u64 timebase = 0; -void __cpuinit rtas_give_timebase(void) +void rtas_give_timebase(void) { unsigned long flags; @@ -1209,7 +1209,7 @@ void __cpuinit rtas_give_timebase(void) local_irq_restore(flags); } -void __cpuinit rtas_take_timebase(void) +void rtas_take_timebase(void) { while (!timebase) barrier(); diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index c5c6407795491..6465fd9b6ce93 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -480,7 +480,7 @@ static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle) secondary_ti = current_set[cpu] = ti; } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { int rc, c; @@ -610,7 +610,7 @@ static struct device_node *cpu_to_l2cache(int cpu) } /* Activate a secondary processor. */ -__cpuinit void start_secondary(void *unused) +void start_secondary(void *unused) { unsigned int cpu = smp_processor_id(); struct device_node *l2_cache; diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c index a15fd1a0690c7..b4e667663d9bf 100644 --- a/arch/powerpc/kernel/sysfs.c +++ b/arch/powerpc/kernel/sysfs.c @@ -352,7 +352,7 @@ static struct device_attribute pa6t_attrs[] = { #endif /* HAS_PPC_PMC_PA6T */ #endif /* HAS_PPC_PMC_CLASSIC */ -static void __cpuinit register_cpu_online(unsigned int cpu) +static void register_cpu_online(unsigned int cpu) { struct cpu *c = &per_cpu(cpu_devices, cpu); struct device *s = &c->dev; @@ -516,7 +516,7 @@ ssize_t arch_cpu_release(const char *buf, size_t count) #endif /* CONFIG_HOTPLUG_CPU */ -static int __cpuinit sysfs_cpu_notify(struct notifier_block *self, +static int sysfs_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned int)(long)hcpu; @@ -536,7 +536,7 @@ static int __cpuinit sysfs_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata sysfs_cpu_nb = { +static struct notifier_block sysfs_cpu_nb = { .notifier_call = sysfs_cpu_notify, }; diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 57fd5c1e8e897..116b384aa7aa3 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -631,7 +631,7 @@ static int __init get_freq(char *name, int cells, unsigned long *val) return found; } -/* should become __cpuinit when secondary_cpu_time_init also is */ +/* should become when secondary_cpu_time_init also is */ void start_cpu_decrementer(void) { #if defined(CONFIG_BOOKE) || defined(CONFIG_40x) diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c index d4f463ac65b1e..1d9c92621b36f 100644 --- a/arch/powerpc/kernel/vdso.c +++ b/arch/powerpc/kernel/vdso.c @@ -711,7 +711,7 @@ static void __init vdso_setup_syscall_map(void) } #ifdef CONFIG_PPC64 -int __cpuinit vdso_getcpu_init(void) +int vdso_getcpu_init(void) { unsigned long cpu, node, val; diff --git a/arch/powerpc/mm/44x_mmu.c b/arch/powerpc/mm/44x_mmu.c index 2c9441ee6bb85..82b1ff759e26d 100644 --- a/arch/powerpc/mm/44x_mmu.c +++ b/arch/powerpc/mm/44x_mmu.c @@ -41,7 +41,7 @@ int icache_44x_need_flush; unsigned long tlb_47x_boltmap[1024/8]; -static void __cpuinit ppc44x_update_tlb_hwater(void) +static void ppc44x_update_tlb_hwater(void) { extern unsigned int tlb_44x_patch_hwater_D[]; extern unsigned int tlb_44x_patch_hwater_I[]; @@ -134,7 +134,7 @@ static void __init ppc47x_update_boltmap(void) /* * "Pins" a 256MB TLB entry in AS0 for kernel lowmem for 47x type MMU */ -static void __cpuinit ppc47x_pin_tlb(unsigned int virt, unsigned int phys) +static void ppc47x_pin_tlb(unsigned int virt, unsigned int phys) { unsigned int rA; int bolted; @@ -229,7 +229,7 @@ void setup_initial_memory_limit(phys_addr_t first_memblock_base, } #ifdef CONFIG_SMP -void __cpuinit mmu_init_secondary(int cpu) +void mmu_init_secondary(int cpu) { unsigned long addr; unsigned long memstart = memstart_addr & ~(PPC_PIN_SIZE - 1); diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index e303a6d74e3a7..4481172664dab 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -807,7 +807,7 @@ void __init early_init_mmu(void) } #ifdef CONFIG_SMP -void __cpuinit early_init_mmu_secondary(void) +void early_init_mmu_secondary(void) { /* Initialize hash table for that CPU */ if (!firmware_has_feature(FW_FEATURE_LPAR)) diff --git a/arch/powerpc/mm/mmu_context_nohash.c b/arch/powerpc/mm/mmu_context_nohash.c index e779642c25e5e..6a7238d68b2f6 100644 --- a/arch/powerpc/mm/mmu_context_nohash.c +++ b/arch/powerpc/mm/mmu_context_nohash.c @@ -329,7 +329,7 @@ void destroy_context(struct mm_struct *mm) #ifdef CONFIG_SMP -static int __cpuinit mmu_context_cpu_notify(struct notifier_block *self, +static int mmu_context_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned int)(long)hcpu; @@ -363,7 +363,7 @@ static int __cpuinit mmu_context_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata mmu_context_cpu_nb = { +static struct notifier_block mmu_context_cpu_nb = { .notifier_call = mmu_context_cpu_notify, }; diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c index 08c6f3185d45b..9ddf2e065ab0e 100644 --- a/arch/powerpc/mm/numa.c +++ b/arch/powerpc/mm/numa.c @@ -532,7 +532,7 @@ static int of_drconf_to_nid_single(struct of_drconf_cell *drmem, * Figure out to which domain a cpu belongs and stick it there. * Return the id of the domain used. */ -static int __cpuinit numa_setup_cpu(unsigned long lcpu) +static int numa_setup_cpu(unsigned long lcpu) { int nid; struct device_node *cpu; @@ -567,7 +567,7 @@ static int __cpuinit numa_setup_cpu(unsigned long lcpu) return nid; } -static int __cpuinit cpu_numa_callback(struct notifier_block *nfb, +static int cpu_numa_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { @@ -948,7 +948,7 @@ static void __init *careful_zallocation(int nid, unsigned long size, return ret; } -static struct notifier_block __cpuinitdata ppc64_numa_nb = { +static struct notifier_block ppc64_numa_nb = { .notifier_call = cpu_numa_callback, .priority = 1 /* Must run before sched domains notifier. */ }; diff --git a/arch/powerpc/mm/tlb_nohash.c b/arch/powerpc/mm/tlb_nohash.c index 6888cad5103de..41cd68dee6816 100644 --- a/arch/powerpc/mm/tlb_nohash.c +++ b/arch/powerpc/mm/tlb_nohash.c @@ -648,7 +648,7 @@ void __init early_init_mmu(void) __early_init_mmu(1); } -void __cpuinit early_init_mmu_secondary(void) +void early_init_mmu_secondary(void) { __early_init_mmu(0); } diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index b63dc809596d5..e4a99f12e9937 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -1825,7 +1825,7 @@ static void power_pmu_setup(int cpu) cpuhw->mmcr[0] = MMCR0_FC; } -static int __cpuinit +static int power_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (long)hcpu; @@ -1842,7 +1842,7 @@ power_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu return NOTIFY_OK; } -int __cpuinit register_power_pmu(struct power_pmu *pmu) +int register_power_pmu(struct power_pmu *pmu) { if (ppmu) return -EBUSY; /* something's already registered */ diff --git a/arch/powerpc/platforms/44x/currituck.c b/arch/powerpc/platforms/44x/currituck.c index ecd3890c40d72..9cef9d381b05c 100644 --- a/arch/powerpc/platforms/44x/currituck.c +++ b/arch/powerpc/platforms/44x/currituck.c @@ -91,12 +91,12 @@ static void __init ppc47x_init_irq(void) } #ifdef CONFIG_SMP -static void __cpuinit smp_ppc47x_setup_cpu(int cpu) +static void smp_ppc47x_setup_cpu(int cpu) { mpic_setup_this_cpu(); } -static int __cpuinit smp_ppc47x_kick_cpu(int cpu) +static int smp_ppc47x_kick_cpu(int cpu) { struct device_node *cpunode = of_get_cpu_node(cpu, NULL); const u64 *spin_table_addr_prop; diff --git a/arch/powerpc/platforms/44x/iss4xx.c b/arch/powerpc/platforms/44x/iss4xx.c index a28a8629727ee..4241bc825800d 100644 --- a/arch/powerpc/platforms/44x/iss4xx.c +++ b/arch/powerpc/platforms/44x/iss4xx.c @@ -81,12 +81,12 @@ static void __init iss4xx_init_irq(void) } #ifdef CONFIG_SMP -static void __cpuinit smp_iss4xx_setup_cpu(int cpu) +static void smp_iss4xx_setup_cpu(int cpu) { mpic_setup_this_cpu(); } -static int __cpuinit smp_iss4xx_kick_cpu(int cpu) +static int smp_iss4xx_kick_cpu(int cpu) { struct device_node *cpunode = of_get_cpu_node(cpu, NULL); const u64 *spin_table_addr_prop; diff --git a/arch/powerpc/platforms/85xx/smp.c b/arch/powerpc/platforms/85xx/smp.c index 6a1759939c6ba..5ced4f5bb2b2e 100644 --- a/arch/powerpc/platforms/85xx/smp.c +++ b/arch/powerpc/platforms/85xx/smp.c @@ -99,7 +99,7 @@ static void mpc85xx_take_timebase(void) } #ifdef CONFIG_HOTPLUG_CPU -static void __cpuinit smp_85xx_mach_cpu_die(void) +static void smp_85xx_mach_cpu_die(void) { unsigned int cpu = smp_processor_id(); u32 tmp; @@ -141,7 +141,7 @@ static inline u32 read_spin_table_addr_l(void *spin_table) return in_be32(&((struct epapr_spin_table *)spin_table)->addr_l); } -static int __cpuinit smp_85xx_kick_cpu(int nr) +static int smp_85xx_kick_cpu(int nr) { unsigned long flags; const u64 *cpu_rel_addr; @@ -362,7 +362,7 @@ static void mpc85xx_smp_machine_kexec(struct kimage *image) } #endif /* CONFIG_KEXEC */ -static void __cpuinit smp_85xx_setup_cpu(int cpu_nr) +static void smp_85xx_setup_cpu(int cpu_nr) { if (smp_85xx_ops.probe == smp_mpic_probe) mpic_setup_this_cpu(); diff --git a/arch/powerpc/platforms/powermac/smp.c b/arch/powerpc/platforms/powermac/smp.c index bdb738a69e418..49c9f9501c213 100644 --- a/arch/powerpc/platforms/powermac/smp.c +++ b/arch/powerpc/platforms/powermac/smp.c @@ -885,7 +885,7 @@ static int smp_core99_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata smp_core99_cpu_nb = { +static struct notifier_block smp_core99_cpu_nb = { .notifier_call = smp_core99_cpu_notify, }; #endif /* CONFIG_HOTPLUG_CPU */ diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c index 88c9459c3e071..c22b2b3d9d84f 100644 --- a/arch/powerpc/platforms/powernv/smp.c +++ b/arch/powerpc/platforms/powernv/smp.c @@ -40,7 +40,7 @@ #define DBG(fmt...) #endif -static void __cpuinit pnv_smp_setup_cpu(int cpu) +static void pnv_smp_setup_cpu(int cpu) { if (cpu != boot_cpuid) xics_setup_cpu(); diff --git a/arch/s390/kernel/cache.c b/arch/s390/kernel/cache.c index 64b24650e4f8c..ae32bfca2f3d1 100644 --- a/arch/s390/kernel/cache.c +++ b/arch/s390/kernel/cache.c @@ -173,7 +173,7 @@ static void __init cache_build_info(void) } } -static struct cache_dir *__cpuinit cache_create_cache_dir(int cpu) +static struct cache_dir *cache_create_cache_dir(int cpu) { struct cache_dir *cache_dir; struct kobject *kobj = NULL; @@ -289,7 +289,7 @@ static struct kobj_type cache_index_type = { .default_attrs = cache_index_default_attrs, }; -static int __cpuinit cache_create_index_dir(struct cache_dir *cache_dir, +static int cache_create_index_dir(struct cache_dir *cache_dir, struct cache *cache, int index, int cpu) { @@ -313,7 +313,7 @@ static int __cpuinit cache_create_index_dir(struct cache_dir *cache_dir, return rc; } -static int __cpuinit cache_add_cpu(int cpu) +static int cache_add_cpu(int cpu) { struct cache_dir *cache_dir; struct cache *cache; @@ -335,7 +335,7 @@ static int __cpuinit cache_add_cpu(int cpu) return 0; } -static void __cpuinit cache_remove_cpu(int cpu) +static void cache_remove_cpu(int cpu) { struct cache_index_dir *index, *next; struct cache_dir *cache_dir; @@ -354,7 +354,7 @@ static void __cpuinit cache_remove_cpu(int cpu) cache_dir_cpu[cpu] = NULL; } -static int __cpuinit cache_hotplug(struct notifier_block *nfb, +static int cache_hotplug(struct notifier_block *nfb, unsigned long action, void *hcpu) { int cpu = (long)hcpu; diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 390d9ae57bb2d..e7518bb6c3130 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -639,7 +639,7 @@ static struct pmu cpumf_pmu = { .cancel_txn = cpumf_pmu_cancel_txn, }; -static int __cpuinit cpumf_pmu_notifier(struct notifier_block *self, +static int cpumf_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (long) hcpu; diff --git a/arch/s390/kernel/processor.c b/arch/s390/kernel/processor.c index 753c41d0ffd3a..24612029f4502 100644 --- a/arch/s390/kernel/processor.c +++ b/arch/s390/kernel/processor.c @@ -21,7 +21,7 @@ static DEFINE_PER_CPU(struct cpuid, cpu_id); /* * cpu_init - initializes state that is per-CPU. */ -void __cpuinit cpu_init(void) +void cpu_init(void) { struct s390_idle_data *idle = &__get_cpu_var(s390_idle); struct cpuid *id = &__get_cpu_var(cpu_id); diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 14647fe09d0cd..4266590ff7012 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -166,7 +166,7 @@ static void pcpu_ec_call(struct pcpu *pcpu, int ec_bit) pcpu_sigp_retry(pcpu, order, 0); } -static int __cpuinit pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) +static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) { struct _lowcore *lc; @@ -619,9 +619,9 @@ static struct sclp_cpu_info *smp_get_cpu_info(void) return info; } -static int __cpuinit smp_add_present_cpu(int cpu); +static int smp_add_present_cpu(int cpu); -static int __cpuinit __smp_rescan_cpus(struct sclp_cpu_info *info, +static int __smp_rescan_cpus(struct sclp_cpu_info *info, int sysfs_add) { struct pcpu *pcpu; @@ -688,7 +688,7 @@ static void __init smp_detect_cpus(void) /* * Activate a secondary processor. */ -static void __cpuinit smp_start_secondary(void *cpuvoid) +static void smp_start_secondary(void *cpuvoid) { S390_lowcore.last_update_clock = get_tod_clock(); S390_lowcore.restart_stack = (unsigned long) restart_stack; @@ -711,7 +711,7 @@ static void __cpuinit smp_start_secondary(void *cpuvoid) } /* Upping and downing of CPUs */ -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { struct pcpu *pcpu; int rc; @@ -967,7 +967,7 @@ static struct attribute_group cpu_online_attr_group = { .attrs = cpu_online_attrs, }; -static int __cpuinit smp_cpu_notify(struct notifier_block *self, +static int smp_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned int)(long)hcpu; @@ -986,7 +986,7 @@ static int __cpuinit smp_cpu_notify(struct notifier_block *self, return notifier_from_errno(err); } -static int __cpuinit smp_add_present_cpu(int cpu) +static int smp_add_present_cpu(int cpu) { struct cpu *c = &pcpu_devices[cpu].cpu; struct device *s = &c->dev; diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c index 62f89d98e8807..811f542b8ed4a 100644 --- a/arch/s390/kernel/sysinfo.c +++ b/arch/s390/kernel/sysinfo.c @@ -418,7 +418,7 @@ void s390_adjust_jiffies(void) /* * calibrate the delay loop */ -void __cpuinit calibrate_delay(void) +void calibrate_delay(void) { s390_adjust_jiffies(); /* Print the good old Bogomips line .. */ diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index 737d50caa4fea..a37d56a338395 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -371,13 +371,13 @@ EXPORT_SYMBOL(del_virt_timer); /* * Start the virtual CPU timer on the current CPU. */ -void __cpuinit init_cpu_vtimer(void) +void init_cpu_vtimer(void) { /* set initial cpu timer */ set_vtimer(VTIMER_MAX_SLICE); } -static int __cpuinit s390_nohz_notify(struct notifier_block *self, +static int s390_nohz_notify(struct notifier_block *self, unsigned long action, void *hcpu) { struct s390_idle_data *idle; diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index d214321db727f..245cfa9b3a670 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -647,7 +647,7 @@ static void pfault_interrupt(struct ext_code ext_code, put_task_struct(tsk); } -static int __cpuinit pfault_cpu_notify(struct notifier_block *self, +static int pfault_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { struct thread_struct *thread, *next; diff --git a/arch/score/mm/tlb-score.c b/arch/score/mm/tlb-score.c index 6fdb100244c81..004073717de03 100644 --- a/arch/score/mm/tlb-score.c +++ b/arch/score/mm/tlb-score.c @@ -240,7 +240,7 @@ void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte) local_irq_restore(flags); } -void __cpuinit tlb_init(void) +void tlb_init(void) { tlblock_set(0); local_flush_tlb_all(); diff --git a/arch/sh/kernel/cpu/init.c b/arch/sh/kernel/cpu/init.c index 61a07dafcd46c..0a2ef460e19d7 100644 --- a/arch/sh/kernel/cpu/init.c +++ b/arch/sh/kernel/cpu/init.c @@ -43,9 +43,9 @@ * peripherals (nofpu, nodsp, and so forth). */ #define onchip_setup(x) \ -static int x##_disabled __cpuinitdata = !cpu_has_##x; \ +static int x##_disabled = !cpu_has_##x; \ \ -static int __cpuinit x##_setup(char *opts) \ +static int x##_setup(char *opts) \ { \ x##_disabled = 1; \ return 1; \ @@ -59,7 +59,7 @@ onchip_setup(dsp); #define CPUOPM 0xff2f0000 #define CPUOPM_RABD (1 << 5) -static void __cpuinit speculative_execution_init(void) +static void speculative_execution_init(void) { /* Clear RABD */ __raw_writel(__raw_readl(CPUOPM) & ~CPUOPM_RABD, CPUOPM); @@ -78,7 +78,7 @@ static void __cpuinit speculative_execution_init(void) #define EXPMASK_BRDSSLP (1 << 1) #define EXPMASK_MMCAW (1 << 4) -static void __cpuinit expmask_init(void) +static void expmask_init(void) { unsigned long expmask = __raw_readl(EXPMASK); @@ -217,7 +217,7 @@ static void detect_cache_shape(void) l2_cache_shape = -1; /* No S-cache */ } -static void __cpuinit fpu_init(void) +static void fpu_init(void) { /* Disable the FPU */ if (fpu_disabled && (current_cpu_data.flags & CPU_HAS_FPU)) { @@ -230,7 +230,7 @@ static void __cpuinit fpu_init(void) } #ifdef CONFIG_SH_DSP -static void __cpuinit release_dsp(void) +static void release_dsp(void) { unsigned long sr; @@ -244,7 +244,7 @@ static void __cpuinit release_dsp(void) ); } -static void __cpuinit dsp_init(void) +static void dsp_init(void) { unsigned long sr; @@ -276,7 +276,7 @@ static void __cpuinit dsp_init(void) release_dsp(); } #else -static inline void __cpuinit dsp_init(void) { } +static inline void dsp_init(void) { } #endif /* CONFIG_SH_DSP */ /** @@ -295,7 +295,7 @@ static inline void __cpuinit dsp_init(void) { } * Each processor family is still responsible for doing its own probing * and cache configuration in cpu_probe(). */ -asmlinkage void __cpuinit cpu_init(void) +asmlinkage void cpu_init(void) { current_thread_info()->cpu = hard_smp_processor_id(); diff --git a/arch/sh/kernel/cpu/sh2/probe.c b/arch/sh/kernel/cpu/sh2/probe.c index bab8e75958aee..6c687ae812ef6 100644 --- a/arch/sh/kernel/cpu/sh2/probe.c +++ b/arch/sh/kernel/cpu/sh2/probe.c @@ -13,7 +13,7 @@ #include #include -void __cpuinit cpu_probe(void) +void cpu_probe(void) { #if defined(CONFIG_CPU_SUBTYPE_SH7619) boot_cpu_data.type = CPU_SH7619; diff --git a/arch/sh/kernel/cpu/sh2a/probe.c b/arch/sh/kernel/cpu/sh2a/probe.c index 5170b6aa4129e..3f87971082f11 100644 --- a/arch/sh/kernel/cpu/sh2a/probe.c +++ b/arch/sh/kernel/cpu/sh2a/probe.c @@ -13,7 +13,7 @@ #include #include -void __cpuinit cpu_probe(void) +void cpu_probe(void) { boot_cpu_data.family = CPU_FAMILY_SH2A; diff --git a/arch/sh/kernel/cpu/sh3/probe.c b/arch/sh/kernel/cpu/sh3/probe.c index bf23c322e164b..426e1e1dcedc7 100644 --- a/arch/sh/kernel/cpu/sh3/probe.c +++ b/arch/sh/kernel/cpu/sh3/probe.c @@ -16,7 +16,7 @@ #include #include -void __cpuinit cpu_probe(void) +void cpu_probe(void) { unsigned long addr0, addr1, data0, data1, data2, data3; diff --git a/arch/sh/kernel/cpu/sh4/probe.c b/arch/sh/kernel/cpu/sh4/probe.c index 0fbbd50bc8ad3..a521bcf506956 100644 --- a/arch/sh/kernel/cpu/sh4/probe.c +++ b/arch/sh/kernel/cpu/sh4/probe.c @@ -15,7 +15,7 @@ #include #include -void __cpuinit cpu_probe(void) +void cpu_probe(void) { unsigned long pvr, prr, cvr; unsigned long size; diff --git a/arch/sh/kernel/cpu/sh4a/smp-shx3.c b/arch/sh/kernel/cpu/sh4a/smp-shx3.c index 03f2b55757cfc..4a298808789c4 100644 --- a/arch/sh/kernel/cpu/sh4a/smp-shx3.c +++ b/arch/sh/kernel/cpu/sh4a/smp-shx3.c @@ -124,7 +124,7 @@ static void shx3_update_boot_vector(unsigned int cpu) __raw_writel(STBCR_RESET, STBCR_REG(cpu)); } -static int __cpuinit +static int shx3_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned int)hcpu; @@ -143,11 +143,11 @@ shx3_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) return NOTIFY_OK; } -static struct notifier_block __cpuinitdata shx3_cpu_notifier = { +static struct notifier_block shx3_cpu_notifier = { .notifier_call = shx3_cpu_callback, }; -static int __cpuinit register_shx3_cpu_notifier(void) +static int register_shx3_cpu_notifier(void) { register_hotcpu_notifier(&shx3_cpu_notifier); return 0; diff --git a/arch/sh/kernel/cpu/sh5/probe.c b/arch/sh/kernel/cpu/sh5/probe.c index 9e882409e4e98..eca427c2f2f3d 100644 --- a/arch/sh/kernel/cpu/sh5/probe.c +++ b/arch/sh/kernel/cpu/sh5/probe.c @@ -17,7 +17,7 @@ #include #include -void __cpuinit cpu_probe(void) +void cpu_probe(void) { unsigned long long cir; diff --git a/arch/sh/kernel/perf_event.c b/arch/sh/kernel/perf_event.c index 068b8a2759b52..b9cefebda55c0 100644 --- a/arch/sh/kernel/perf_event.c +++ b/arch/sh/kernel/perf_event.c @@ -367,7 +367,7 @@ static void sh_pmu_setup(int cpu) memset(cpuhw, 0, sizeof(struct cpu_hw_events)); } -static int __cpuinit +static int sh_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (long)hcpu; @@ -384,7 +384,7 @@ sh_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) return NOTIFY_OK; } -int __cpuinit register_sh_pmu(struct sh_pmu *_pmu) +int register_sh_pmu(struct sh_pmu *_pmu) { if (sh_pmu) return -EBUSY; diff --git a/arch/sh/kernel/process.c b/arch/sh/kernel/process.c index 055d91b70305f..53bc6c4c84ecd 100644 --- a/arch/sh/kernel/process.c +++ b/arch/sh/kernel/process.c @@ -65,7 +65,7 @@ void arch_task_cache_init(void) # define HAVE_SOFTFP 0 #endif -void __cpuinit init_thread_xstate(void) +void init_thread_xstate(void) { if (boot_cpu_data.flags & CPU_HAS_FPU) xstate_size = sizeof(struct sh_fpu_hard_struct); diff --git a/arch/sh/kernel/setup.c b/arch/sh/kernel/setup.c index ebe7a7d97215e..1cf90e947dbf1 100644 --- a/arch/sh/kernel/setup.c +++ b/arch/sh/kernel/setup.c @@ -172,7 +172,7 @@ void __init check_for_initrd(void) #endif } -void __cpuinit calibrate_delay(void) +void calibrate_delay(void) { struct clk *clk = clk_get(NULL, "cpu_clk"); diff --git a/arch/sh/kernel/smp.c b/arch/sh/kernel/smp.c index 45696451f0ea3..86a7936a980b6 100644 --- a/arch/sh/kernel/smp.c +++ b/arch/sh/kernel/smp.c @@ -37,7 +37,7 @@ struct plat_smp_ops *mp_ops = NULL; /* State of each CPU */ DEFINE_PER_CPU(int, cpu_state) = { 0 }; -void __cpuinit register_smp_ops(struct plat_smp_ops *ops) +void register_smp_ops(struct plat_smp_ops *ops) { if (mp_ops) printk(KERN_WARNING "Overriding previously set SMP ops\n"); @@ -45,7 +45,7 @@ void __cpuinit register_smp_ops(struct plat_smp_ops *ops) mp_ops = ops; } -static inline void __cpuinit smp_store_cpu_info(unsigned int cpu) +static inline void smp_store_cpu_info(unsigned int cpu) { struct sh_cpuinfo *c = cpu_data + cpu; @@ -174,7 +174,7 @@ void native_play_dead(void) } #endif -asmlinkage void __cpuinit start_secondary(void) +asmlinkage void start_secondary(void) { unsigned int cpu = smp_processor_id(); struct mm_struct *mm = &init_mm; @@ -215,7 +215,7 @@ extern struct { void *thread_info; } stack_start; -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tsk) +int __cpu_up(unsigned int cpu, struct task_struct *tsk) { unsigned long timeout; diff --git a/arch/sh/kernel/traps_32.c b/arch/sh/kernel/traps_32.c index 5f513a64dedfc..68e99f09171dd 100644 --- a/arch/sh/kernel/traps_32.c +++ b/arch/sh/kernel/traps_32.c @@ -741,7 +741,7 @@ asmlinkage void do_exception_error(unsigned long r4, unsigned long r5, die_if_kernel("exception", regs, ex); } -void __cpuinit per_cpu_trap_init(void) +void per_cpu_trap_init(void) { extern void *vbr_base; diff --git a/arch/sh/kernel/traps_64.c b/arch/sh/kernel/traps_64.c index f87d20da17916..112ea11c030d6 100644 --- a/arch/sh/kernel/traps_64.c +++ b/arch/sh/kernel/traps_64.c @@ -810,7 +810,7 @@ asmlinkage void do_debug_interrupt(unsigned long code, struct pt_regs *regs) poke_real_address_q(DM_EXP_CAUSE_PHY, 0x0); } -void __cpuinit per_cpu_trap_init(void) +void per_cpu_trap_init(void) { /* Nothing to do for now, VBR initialization later. */ } diff --git a/arch/sh/mm/tlb-sh5.c b/arch/sh/mm/tlb-sh5.c index ff1c40a31cbc0..e4bb2a8e0a69d 100644 --- a/arch/sh/mm/tlb-sh5.c +++ b/arch/sh/mm/tlb-sh5.c @@ -17,7 +17,7 @@ /** * sh64_tlb_init - Perform initial setup for the DTLB and ITLB. */ -int __cpuinit sh64_tlb_init(void) +int sh64_tlb_init(void) { /* Assign some sane DTLB defaults */ cpu_data->dtlb.entries = 64; diff --git a/arch/sparc/kernel/ds.c b/arch/sparc/kernel/ds.c index 252f8768aac69..3f7d4b372f3a6 100644 --- a/arch/sparc/kernel/ds.c +++ b/arch/sparc/kernel/ds.c @@ -528,7 +528,7 @@ static void dr_cpu_mark(struct ds_data *resp, int cpu, int ncpus, } } -static int __cpuinit dr_cpu_configure(struct ds_info *dp, +static int dr_cpu_configure(struct ds_info *dp, struct ds_cap_state *cp, u64 req_num, cpumask_t *mask) @@ -627,7 +627,7 @@ static int dr_cpu_unconfigure(struct ds_info *dp, return 0; } -static void __cpuinit dr_cpu_data(struct ds_info *dp, +static void dr_cpu_data(struct ds_info *dp, struct ds_cap_state *cp, void *buf, int len) { diff --git a/arch/sparc/kernel/entry.h b/arch/sparc/kernel/entry.h index cc3c5cb47cdaa..9c179fbfb219c 100644 --- a/arch/sparc/kernel/entry.h +++ b/arch/sparc/kernel/entry.h @@ -250,7 +250,7 @@ extern struct ino_bucket *ivector_table; extern unsigned long ivector_table_pa; extern void init_irqwork_curcpu(void); -extern void __cpuinit sun4v_register_mondo_queues(int this_cpu); +extern void sun4v_register_mondo_queues(int this_cpu); #endif /* CONFIG_SPARC32 */ #endif /* _ENTRY_H */ diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c index 9bcbbe2c4e7e1..d16d16520678c 100644 --- a/arch/sparc/kernel/irq_64.c +++ b/arch/sparc/kernel/irq_64.c @@ -835,7 +835,7 @@ void notrace init_irqwork_curcpu(void) * Therefore you cannot make any OBP calls, not even prom_printf, * from these two routines. */ -static void __cpuinit notrace register_one_mondo(unsigned long paddr, unsigned long type, unsigned long qmask) +static void notrace register_one_mondo(unsigned long paddr, unsigned long type, unsigned long qmask) { unsigned long num_entries = (qmask + 1) / 64; unsigned long status; @@ -848,7 +848,7 @@ static void __cpuinit notrace register_one_mondo(unsigned long paddr, unsigned l } } -void __cpuinit notrace sun4v_register_mondo_queues(int this_cpu) +void notrace sun4v_register_mondo_queues(int this_cpu) { struct trap_per_cpu *tb = &trap_block[this_cpu]; diff --git a/arch/sparc/kernel/leon_smp.c b/arch/sparc/kernel/leon_smp.c index 6cfc1b09ec25a..b5945694bef63 100644 --- a/arch/sparc/kernel/leon_smp.c +++ b/arch/sparc/kernel/leon_smp.c @@ -54,7 +54,7 @@ extern ctxd_t *srmmu_ctx_table_phys; static int smp_processors_ready; extern volatile unsigned long cpu_callin_map[NR_CPUS]; extern cpumask_t smp_commenced_mask; -void __cpuinit leon_configure_cache_smp(void); +void leon_configure_cache_smp(void); static void leon_ipi_init(void); /* IRQ number of LEON IPIs */ @@ -69,12 +69,12 @@ static inline unsigned long do_swap(volatile unsigned long *ptr, return val; } -void __cpuinit leon_cpu_pre_starting(void *arg) +void leon_cpu_pre_starting(void *arg) { leon_configure_cache_smp(); } -void __cpuinit leon_cpu_pre_online(void *arg) +void leon_cpu_pre_online(void *arg) { int cpuid = hard_smp_processor_id(); @@ -106,7 +106,7 @@ void __cpuinit leon_cpu_pre_online(void *arg) extern struct linux_prom_registers smp_penguin_ctable; -void __cpuinit leon_configure_cache_smp(void) +void leon_configure_cache_smp(void) { unsigned long cfg = sparc_leon3_get_dcachecfg(); int me = smp_processor_id(); @@ -186,7 +186,7 @@ void __init leon_boot_cpus(void) } -int __cpuinit leon_boot_one_cpu(int i, struct task_struct *idle) +int leon_boot_one_cpu(int i, struct task_struct *idle) { int timeout; diff --git a/arch/sparc/kernel/mdesc.c b/arch/sparc/kernel/mdesc.c index 831c001604e8e..3961054c595c4 100644 --- a/arch/sparc/kernel/mdesc.c +++ b/arch/sparc/kernel/mdesc.c @@ -571,7 +571,7 @@ static void __init report_platform_properties(void) mdesc_release(hp); } -static void __cpuinit fill_in_one_cache(cpuinfo_sparc *c, +static void fill_in_one_cache(cpuinfo_sparc *c, struct mdesc_handle *hp, u64 mp) { @@ -616,7 +616,7 @@ static void __cpuinit fill_in_one_cache(cpuinfo_sparc *c, } } -static void __cpuinit mark_core_ids(struct mdesc_handle *hp, u64 mp, int core_id) +static void mark_core_ids(struct mdesc_handle *hp, u64 mp, int core_id) { u64 a; @@ -649,7 +649,7 @@ static void __cpuinit mark_core_ids(struct mdesc_handle *hp, u64 mp, int core_id } } -static void __cpuinit set_core_ids(struct mdesc_handle *hp) +static void set_core_ids(struct mdesc_handle *hp) { int idx; u64 mp; @@ -674,7 +674,7 @@ static void __cpuinit set_core_ids(struct mdesc_handle *hp) } } -static void __cpuinit mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id) +static void mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id) { u64 a; @@ -693,7 +693,7 @@ static void __cpuinit mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id } } -static void __cpuinit __set_proc_ids(struct mdesc_handle *hp, const char *exec_unit_name) +static void __set_proc_ids(struct mdesc_handle *hp, const char *exec_unit_name) { int idx; u64 mp; @@ -714,13 +714,13 @@ static void __cpuinit __set_proc_ids(struct mdesc_handle *hp, const char *exec_u } } -static void __cpuinit set_proc_ids(struct mdesc_handle *hp) +static void set_proc_ids(struct mdesc_handle *hp) { __set_proc_ids(hp, "exec_unit"); __set_proc_ids(hp, "exec-unit"); } -static void __cpuinit get_one_mondo_bits(const u64 *p, unsigned int *mask, +static void get_one_mondo_bits(const u64 *p, unsigned int *mask, unsigned long def, unsigned long max) { u64 val; @@ -742,7 +742,7 @@ static void __cpuinit get_one_mondo_bits(const u64 *p, unsigned int *mask, *mask = ((1U << def) * 64U) - 1U; } -static void __cpuinit get_mondo_data(struct mdesc_handle *hp, u64 mp, +static void get_mondo_data(struct mdesc_handle *hp, u64 mp, struct trap_per_cpu *tb) { static int printed; @@ -769,7 +769,7 @@ static void __cpuinit get_mondo_data(struct mdesc_handle *hp, u64 mp, } } -static void * __cpuinit mdesc_iterate_over_cpus(void *(*func)(struct mdesc_handle *, u64, int, void *), void *arg, cpumask_t *mask) +static void * mdesc_iterate_over_cpus(void *(*func)(struct mdesc_handle *, u64, int, void *), void *arg, cpumask_t *mask) { struct mdesc_handle *hp = mdesc_grab(); void *ret = NULL; @@ -799,7 +799,7 @@ static void * __cpuinit mdesc_iterate_over_cpus(void *(*func)(struct mdesc_handl return ret; } -static void * __cpuinit record_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, void *arg) +static void * record_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, void *arg) { ncpus_probed++; #ifdef CONFIG_SMP @@ -808,7 +808,7 @@ static void * __cpuinit record_one_cpu(struct mdesc_handle *hp, u64 mp, int cpui return NULL; } -void __cpuinit mdesc_populate_present_mask(cpumask_t *mask) +void mdesc_populate_present_mask(cpumask_t *mask) { if (tlb_type != hypervisor) return; @@ -841,7 +841,7 @@ void __init mdesc_get_page_sizes(cpumask_t *mask, unsigned long *pgsz_mask) mdesc_iterate_over_cpus(check_one_pgsz, pgsz_mask, mask); } -static void * __cpuinit fill_in_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, void *arg) +static void * fill_in_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, void *arg) { const u64 *cfreq = mdesc_get_property(hp, mp, "clock-frequency", NULL); struct trap_per_cpu *tb; @@ -890,7 +890,7 @@ static void * __cpuinit fill_in_one_cpu(struct mdesc_handle *hp, u64 mp, int cpu return NULL; } -void __cpuinit mdesc_fill_in_cpu_data(cpumask_t *mask) +void mdesc_fill_in_cpu_data(cpumask_t *mask) { struct mdesc_handle *hp; diff --git a/arch/sparc/kernel/smp_32.c b/arch/sparc/kernel/smp_32.c index e3f2b81c23f14..a102bfba6ea86 100644 --- a/arch/sparc/kernel/smp_32.c +++ b/arch/sparc/kernel/smp_32.c @@ -39,7 +39,7 @@ #include "kernel.h" #include "irq.h" -volatile unsigned long cpu_callin_map[NR_CPUS] __cpuinitdata = {0,}; +volatile unsigned long cpu_callin_map[NR_CPUS] = {0,}; cpumask_t smp_commenced_mask = CPU_MASK_NONE; @@ -53,7 +53,7 @@ const struct sparc32_ipi_ops *sparc32_ipi_ops; * instruction which is much better... */ -void __cpuinit smp_store_cpu_info(int id) +void smp_store_cpu_info(int id) { int cpu_node; int mid; @@ -120,7 +120,7 @@ void cpu_panic(void) panic("SMP bolixed\n"); } -struct linux_prom_registers smp_penguin_ctable __cpuinitdata = { 0 }; +struct linux_prom_registers smp_penguin_ctable = { 0 }; void smp_send_reschedule(int cpu) { @@ -259,10 +259,10 @@ void __init smp_prepare_boot_cpu(void) set_cpu_possible(cpuid, true); } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { - extern int __cpuinit smp4m_boot_one_cpu(int, struct task_struct *); - extern int __cpuinit smp4d_boot_one_cpu(int, struct task_struct *); + extern int smp4m_boot_one_cpu(int, struct task_struct *); + extern int smp4d_boot_one_cpu(int, struct task_struct *); int ret=0; switch(sparc_cpu_model) { @@ -297,7 +297,7 @@ int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) return ret; } -void __cpuinit arch_cpu_pre_starting(void *arg) +void arch_cpu_pre_starting(void *arg) { local_ops->cache_all(); local_ops->tlb_all(); @@ -317,7 +317,7 @@ void __cpuinit arch_cpu_pre_starting(void *arg) } } -void __cpuinit arch_cpu_pre_online(void *arg) +void arch_cpu_pre_online(void *arg) { unsigned int cpuid = hard_smp_processor_id(); @@ -344,7 +344,7 @@ void __cpuinit arch_cpu_pre_online(void *arg) } } -void __cpuinit sparc_start_secondary(void *arg) +void sparc_start_secondary(void *arg) { unsigned int cpu; @@ -375,7 +375,7 @@ void __cpuinit sparc_start_secondary(void *arg) BUG(); } -void __cpuinit smp_callin(void) +void smp_callin(void) { sparc_start_secondary(NULL); } diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index 173964d5e948a..7c3d82107a61c 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c @@ -87,7 +87,7 @@ extern void setup_sparc64_timer(void); static volatile unsigned long callin_flag = 0; -void __cpuinit smp_callin(void) +void smp_callin(void) { int cpuid = hard_smp_processor_id(); @@ -281,7 +281,7 @@ static unsigned long kimage_addr_to_ra(void *p) return kern_base + (val - KERNBASE); } -static void __cpuinit ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread_reg, void **descrp) +static void ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread_reg, void **descrp) { extern unsigned long sparc64_ttable_tl0; extern unsigned long kern_locked_tte_data; @@ -342,7 +342,7 @@ extern unsigned long sparc64_cpu_startup; */ static struct thread_info *cpu_new_thread = NULL; -static int __cpuinit smp_boot_one_cpu(unsigned int cpu, struct task_struct *idle) +static int smp_boot_one_cpu(unsigned int cpu, struct task_struct *idle) { unsigned long entry = (unsigned long)(&sparc64_cpu_startup); @@ -1270,7 +1270,7 @@ void smp_fill_in_sib_core_maps(void) } } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { int ret = smp_boot_one_cpu(cpu, tidle); diff --git a/arch/sparc/kernel/sun4d_smp.c b/arch/sparc/kernel/sun4d_smp.c index c9eb82f23d928..d5c319553fd08 100644 --- a/arch/sparc/kernel/sun4d_smp.c +++ b/arch/sparc/kernel/sun4d_smp.c @@ -50,7 +50,7 @@ static inline void show_leds(int cpuid) "i" (ASI_M_CTL)); } -void __cpuinit sun4d_cpu_pre_starting(void *arg) +void sun4d_cpu_pre_starting(void *arg) { int cpuid = hard_smp_processor_id(); @@ -62,7 +62,7 @@ void __cpuinit sun4d_cpu_pre_starting(void *arg) cc_set_imsk((cc_get_imsk() & ~0x8000) | 0x4000); } -void __cpuinit sun4d_cpu_pre_online(void *arg) +void sun4d_cpu_pre_online(void *arg) { unsigned long flags; int cpuid; @@ -118,7 +118,7 @@ void __init smp4d_boot_cpus(void) local_ops->cache_all(); } -int __cpuinit smp4d_boot_one_cpu(int i, struct task_struct *idle) +int smp4d_boot_one_cpu(int i, struct task_struct *idle) { unsigned long *entry = &sun4d_cpu_startup; int timeout; diff --git a/arch/sparc/kernel/sun4m_smp.c b/arch/sparc/kernel/sun4m_smp.c index 8a65f158153dd..d3408e72d20c0 100644 --- a/arch/sparc/kernel/sun4m_smp.c +++ b/arch/sparc/kernel/sun4m_smp.c @@ -34,11 +34,11 @@ swap_ulong(volatile unsigned long *ptr, unsigned long val) return val; } -void __cpuinit sun4m_cpu_pre_starting(void *arg) +void sun4m_cpu_pre_starting(void *arg) { } -void __cpuinit sun4m_cpu_pre_online(void *arg) +void sun4m_cpu_pre_online(void *arg) { int cpuid = hard_smp_processor_id(); @@ -75,7 +75,7 @@ void __init smp4m_boot_cpus(void) local_ops->cache_all(); } -int __cpuinit smp4m_boot_one_cpu(int i, struct task_struct *idle) +int smp4m_boot_one_cpu(int i, struct task_struct *idle) { unsigned long *entry = &sun4m_cpu_startup; int timeout; diff --git a/arch/sparc/kernel/sysfs.c b/arch/sparc/kernel/sysfs.c index 654e8aad3bbef..c21c673e5f7cd 100644 --- a/arch/sparc/kernel/sysfs.c +++ b/arch/sparc/kernel/sysfs.c @@ -246,7 +246,7 @@ static void unregister_cpu_online(unsigned int cpu) } #endif -static int __cpuinit sysfs_cpu_notify(struct notifier_block *self, +static int sysfs_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned int)(long)hcpu; @@ -266,7 +266,7 @@ static int __cpuinit sysfs_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata sysfs_cpu_nb = { +static struct notifier_block sysfs_cpu_nb = { .notifier_call = sysfs_cpu_notify, }; diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index 272ebe7d380d4..04541b0cd9833 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c @@ -1698,7 +1698,7 @@ static void __init sun4v_ktsb_init(void) #endif } -void __cpuinit sun4v_ktsb_register(void) +void sun4v_ktsb_register(void) { unsigned long pa, ret; diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index f58cb540ff948..1e0e7d2837da1 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -860,7 +860,7 @@ static void __init map_kernel(void) } } -void (*poke_srmmu)(void) __cpuinitdata = NULL; +void (*poke_srmmu)(void) = NULL; extern unsigned long bootmem_init(unsigned long *pages_avail); @@ -1058,7 +1058,7 @@ static void __init init_vac_layout(void) (int)vac_cache_size, (int)vac_line_size); } -static void __cpuinit poke_hypersparc(void) +static void poke_hypersparc(void) { volatile unsigned long clear; unsigned long mreg = srmmu_get_mmureg(); @@ -1110,7 +1110,7 @@ static void __init init_hypersparc(void) hypersparc_setup_blockops(); } -static void __cpuinit poke_swift(void) +static void poke_swift(void) { unsigned long mreg; @@ -1290,7 +1290,7 @@ static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long } -static void __cpuinit poke_turbosparc(void) +static void poke_turbosparc(void) { unsigned long mreg = srmmu_get_mmureg(); unsigned long ccreg; @@ -1353,7 +1353,7 @@ static void __init init_turbosparc(void) poke_srmmu = poke_turbosparc; } -static void __cpuinit poke_tsunami(void) +static void poke_tsunami(void) { unsigned long mreg = srmmu_get_mmureg(); @@ -1394,7 +1394,7 @@ static void __init init_tsunami(void) tsunami_setup_blockops(); } -static void __cpuinit poke_viking(void) +static void poke_viking(void) { unsigned long mreg = srmmu_get_mmureg(); static int smp_catch; diff --git a/arch/tile/kernel/irq.c b/arch/tile/kernel/irq.c index 02e6280650125..3ccf2cd7182eb 100644 --- a/arch/tile/kernel/irq.c +++ b/arch/tile/kernel/irq.c @@ -220,7 +220,7 @@ void __init init_IRQ(void) ipi_init(); } -void __cpuinit setup_irq_regs(void) +void setup_irq_regs(void) { /* Enable interrupt delivery. */ unmask_irqs(~0UL); diff --git a/arch/tile/kernel/messaging.c b/arch/tile/kernel/messaging.c index 0858ee6b520f6..00331af9525dc 100644 --- a/arch/tile/kernel/messaging.c +++ b/arch/tile/kernel/messaging.c @@ -25,7 +25,7 @@ /* All messages are stored here */ static DEFINE_PER_CPU(HV_MsgState, msg_state); -void __cpuinit init_messaging(void) +void init_messaging(void) { /* Allocate storage for messages in kernel space */ HV_MsgState *state = &__get_cpu_var(msg_state); diff --git a/arch/tile/kernel/setup.c b/arch/tile/kernel/setup.c index 5ebe936762016..0d7c1f30349ea 100644 --- a/arch/tile/kernel/setup.c +++ b/arch/tile/kernel/setup.c @@ -58,8 +58,8 @@ struct pglist_data node_data[MAX_NUMNODES] __read_mostly; EXPORT_SYMBOL(node_data); /* Information on the NUMA nodes that we compute early */ -unsigned long __cpuinitdata node_start_pfn[MAX_NUMNODES]; -unsigned long __cpuinitdata node_end_pfn[MAX_NUMNODES]; +unsigned long node_start_pfn[MAX_NUMNODES]; +unsigned long node_end_pfn[MAX_NUMNODES]; unsigned long __initdata node_memmap_pfn[MAX_NUMNODES]; unsigned long __initdata node_percpu_pfn[MAX_NUMNODES]; unsigned long __initdata node_free_pfn[MAX_NUMNODES]; @@ -84,7 +84,7 @@ unsigned long __initdata boot_pc = (unsigned long)start_kernel; #ifdef CONFIG_HIGHMEM /* Page frame index of end of lowmem on each controller. */ -unsigned long __cpuinitdata node_lowmem_end_pfn[MAX_NUMNODES]; +unsigned long node_lowmem_end_pfn[MAX_NUMNODES]; /* Number of pages that can be mapped into lowmem. */ static unsigned long __initdata mappable_physpages; @@ -290,7 +290,7 @@ static void *__init setup_pa_va_mapping(void) * This is up to 4 mappings for lowmem, one mapping per memory * controller, plus one for our text segment. */ -static void __cpuinit store_permanent_mappings(void) +static void store_permanent_mappings(void) { int i; @@ -935,7 +935,7 @@ subsys_initcall(topology_init); * So the values we set up here in the hypervisor may be overridden on * the boot cpu as arguments are parsed. */ -static __cpuinit void init_super_pages(void) +static void init_super_pages(void) { #ifdef CONFIG_HUGETLB_SUPER_PAGES int i; @@ -950,7 +950,7 @@ static __cpuinit void init_super_pages(void) * * Called from setup_arch() on the boot cpu, or online_secondary(). */ -void __cpuinit setup_cpu(int boot) +void setup_cpu(int boot) { /* The boot cpu sets up its permanent mappings much earlier. */ if (!boot) diff --git a/arch/tile/kernel/smpboot.c b/arch/tile/kernel/smpboot.c index 44bab29bf2f3a..c4d09761a3838 100644 --- a/arch/tile/kernel/smpboot.c +++ b/arch/tile/kernel/smpboot.c @@ -140,7 +140,7 @@ static struct cpumask cpu_started __cpuinitdata; * to this path without knowing what you're doing, since SMP booting * is pretty fragile. */ -static void __cpuinit start_secondary(void) +static void start_secondary(void) { int cpuid = smp_processor_id(); @@ -183,7 +183,7 @@ static void __cpuinit start_secondary(void) /* * Bring a secondary processor online. */ -void __cpuinit online_secondary(void) +void online_secondary(void) { /* * low-memory mappings have been cleared, flush them from @@ -210,7 +210,7 @@ void __cpuinit online_secondary(void) cpu_startup_entry(CPUHP_ONLINE); } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { /* Wait 5s total for all CPUs for them to come online */ static int timeout; diff --git a/arch/tile/kernel/time.c b/arch/tile/kernel/time.c index 9df6d0d6d187e..251c05d50060d 100644 --- a/arch/tile/kernel/time.c +++ b/arch/tile/kernel/time.c @@ -159,7 +159,7 @@ static DEFINE_PER_CPU(struct clock_event_device, tile_timer) = { .set_mode = tile_timer_set_mode, }; -void __cpuinit setup_tile_timer(void) +void setup_tile_timer(void) { struct clock_event_device *evt = &__get_cpu_var(tile_timer); diff --git a/arch/x86/include/asm/cpu.h b/arch/x86/include/asm/cpu.h index 5f9a1243190e8..d2b12988d2ed1 100644 --- a/arch/x86/include/asm/cpu.h +++ b/arch/x86/include/asm/cpu.h @@ -28,7 +28,7 @@ struct x86_cpu { #ifdef CONFIG_HOTPLUG_CPU extern int arch_register_cpu(int num); extern void arch_unregister_cpu(int); -extern void __cpuinit start_cpu0(void); +extern void start_cpu0(void); #ifdef CONFIG_DEBUG_HOTPLUG_CPU0 extern int _debug_hotplug_cpu(int cpu, int action); #endif diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h index 6bc3985ee473d..f98bd66253182 100644 --- a/arch/x86/include/asm/microcode.h +++ b/arch/x86/include/asm/microcode.h @@ -60,11 +60,11 @@ static inline void __exit exit_amd_microcode(void) {} #ifdef CONFIG_MICROCODE_EARLY #define MAX_UCODE_COUNT 128 extern void __init load_ucode_bsp(void); -extern void __cpuinit load_ucode_ap(void); +extern void load_ucode_ap(void); extern int __init save_microcode_in_initrd(void); #else static inline void __init load_ucode_bsp(void) {} -static inline void __cpuinit load_ucode_ap(void) {} +static inline void load_ucode_ap(void) {} static inline int __init save_microcode_in_initrd(void) { return 0; diff --git a/arch/x86/include/asm/microcode_intel.h b/arch/x86/include/asm/microcode_intel.h index 5356f927d411c..62c1f79625777 100644 --- a/arch/x86/include/asm/microcode_intel.h +++ b/arch/x86/include/asm/microcode_intel.h @@ -65,11 +65,11 @@ update_match_revision(struct microcode_header_intel *mc_header, int rev); #ifdef CONFIG_MICROCODE_INTEL_EARLY extern void __init load_ucode_intel_bsp(void); -extern void __cpuinit load_ucode_intel_ap(void); +extern void load_ucode_intel_ap(void); extern void show_ucode_info_early(void); #else static inline __init void load_ucode_intel_bsp(void) {} -static inline __cpuinit void load_ucode_intel_ap(void) {} +static inline void load_ucode_intel_ap(void) {} static inline void show_ucode_info_early(void) {} #endif diff --git a/arch/x86/include/asm/mmconfig.h b/arch/x86/include/asm/mmconfig.h index 9b119da1d105f..04a3fed22cfe0 100644 --- a/arch/x86/include/asm/mmconfig.h +++ b/arch/x86/include/asm/mmconfig.h @@ -2,8 +2,8 @@ #define _ASM_X86_MMCONFIG_H #ifdef CONFIG_PCI_MMCONFIG -extern void __cpuinit fam10h_check_enable_mmcfg(void); -extern void __cpuinit check_enable_amd_mmconf_dmi(void); +extern void fam10h_check_enable_mmcfg(void); +extern void check_enable_amd_mmconf_dmi(void); #else static inline void fam10h_check_enable_mmcfg(void) { } static inline void check_enable_amd_mmconf_dmi(void) { } diff --git a/arch/x86/include/asm/mpspec.h b/arch/x86/include/asm/mpspec.h index 3e2f42a4b872a..626cf70082d7c 100644 --- a/arch/x86/include/asm/mpspec.h +++ b/arch/x86/include/asm/mpspec.h @@ -94,7 +94,7 @@ static inline void early_reserve_e820_mpc_new(void) { } #define default_get_smp_config x86_init_uint_noop #endif -void __cpuinit generic_processor_info(int apicid, int version); +void generic_processor_info(int apicid, int version); #ifdef CONFIG_ACPI extern void mp_register_ioapic(int id, u32 address, u32 gsi_base); extern void mp_override_legacy_irq(u8 bus_irq, u8 polarity, u8 trigger, diff --git a/arch/x86/include/asm/numa.h b/arch/x86/include/asm/numa.h index 1b99ee5c9f00d..4064acae625d9 100644 --- a/arch/x86/include/asm/numa.h +++ b/arch/x86/include/asm/numa.h @@ -39,7 +39,7 @@ static inline void set_apicid_to_node(int apicid, s16 node) __apicid_to_node[apicid] = node; } -extern int __cpuinit numa_cpu_node(int cpu); +extern int numa_cpu_node(int cpu); #else /* CONFIG_NUMA */ static inline void set_apicid_to_node(int apicid, s16 node) @@ -60,8 +60,8 @@ static inline int numa_cpu_node(int cpu) extern void numa_set_node(int cpu, int node); extern void numa_clear_node(int cpu); extern void __init init_cpu_to_node(void); -extern void __cpuinit numa_add_cpu(int cpu); -extern void __cpuinit numa_remove_cpu(int cpu); +extern void numa_add_cpu(int cpu); +extern void numa_remove_cpu(int cpu); #else /* CONFIG_NUMA */ static inline void numa_set_node(int cpu, int node) { } static inline void numa_clear_node(int cpu) { } diff --git a/arch/x86/include/asm/prom.h b/arch/x86/include/asm/prom.h index 60bef663609a4..bade6ac3b14fc 100644 --- a/arch/x86/include/asm/prom.h +++ b/arch/x86/include/asm/prom.h @@ -27,7 +27,7 @@ extern int of_ioapic; extern u64 initial_dtb; extern void add_dtb(u64 data); extern void x86_add_irq_domains(void); -void __cpuinit x86_of_pci_init(void); +void x86_of_pci_init(void); void x86_dtb_init(void); #else static inline void add_dtb(u64 data) { } diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index b073aaea747c2..1e2b33829cb2b 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -179,7 +179,7 @@ static inline int wbinvd_on_all_cpus(void) } #endif /* CONFIG_SMP */ -extern unsigned disabled_cpus __cpuinitdata; +extern unsigned disabled_cpusdata; #ifdef CONFIG_X86_32_SMP /* diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 230c8ea878e5f..6c96b60fe3f39 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -194,7 +194,7 @@ static int __init acpi_parse_madt(struct acpi_table_header *table) return 0; } -static void __cpuinit acpi_register_lapic(int id, u8 enabled) +static void acpi_register_lapic(int id, u8 enabled) { unsigned int ver = 0; @@ -600,7 +600,7 @@ void __init acpi_set_irq_model_ioapic(void) #ifdef CONFIG_ACPI_HOTPLUG_CPU #include -static void __cpuinit acpi_map_cpu2node(acpi_handle handle, int cpu, int physid) +static void acpi_map_cpu2node(acpi_handle handle, int cpu, int physid) { #ifdef CONFIG_ACPI_NUMA int nid; @@ -613,7 +613,7 @@ static void __cpuinit acpi_map_cpu2node(acpi_handle handle, int cpu, int physid) #endif } -static int __cpuinit _acpi_map_lsapic(acpi_handle handle, int *pcpu) +static int _acpi_map_lsapic(acpi_handle handle, int *pcpu) { struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; union acpi_object *obj; diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 3cd8bfc3c4b69..17a5c0863a45f 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -550,7 +550,7 @@ static DEFINE_PER_CPU(struct clock_event_device, lapic_events); * Setup the local APIC timer for this CPU. Copy the initialized values * of the boot CPU and register the clock event in the framework. */ -static void __cpuinit setup_APIC_timer(void) +static void setup_APIC_timer(void) { struct clock_event_device *levt = &__get_cpu_var(lapic_events); @@ -872,7 +872,7 @@ void __init setup_boot_APIC_clock(void) setup_APIC_timer(); } -void __cpuinit setup_secondary_APIC_clock(void) +void setup_secondary_APIC_clock(void) { setup_APIC_timer(); } @@ -1217,7 +1217,7 @@ void __init init_bsp_APIC(void) apic_write(APIC_LVT1, value); } -static void __cpuinit lapic_setup_esr(void) +static void lapic_setup_esr(void) { unsigned int oldvalue, value, maxlvt; @@ -1264,7 +1264,7 @@ static void __cpuinit lapic_setup_esr(void) * Used to setup local APIC while initializing BSP or bringin up APs. * Always called with preemption disabled. */ -void __cpuinit setup_local_APIC(void) +void setup_local_APIC(void) { int cpu = smp_processor_id(); unsigned int value, queued; @@ -1459,7 +1459,7 @@ void __cpuinit setup_local_APIC(void) #endif } -void __cpuinit end_local_APIC_setup(void) +void end_local_APIC_setup(void) { lapic_setup_esr(); @@ -2072,7 +2072,7 @@ void disconnect_bsp_APIC(int virt_wire_setup) apic_write(APIC_LVT1, value); } -void __cpuinit generic_processor_info(int apicid, int version) +void generic_processor_info(int apicid, int version) { int cpu, max = nr_cpu_ids; bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid, @@ -2342,7 +2342,7 @@ static struct syscore_ops lapic_syscore_ops = { .suspend = lapic_suspend, }; -static void __cpuinit apic_pm_activate(void) +static void apic_pm_activate(void) { apic_pm_state.active = 1; } @@ -2367,7 +2367,7 @@ static void apic_pm_activate(void) { } #ifdef CONFIG_X86_64 -static int __cpuinit apic_cluster_num(void) +static int apic_cluster_num(void) { int i, clusters, zeros; unsigned id; @@ -2412,10 +2412,10 @@ static int __cpuinit apic_cluster_num(void) return clusters; } -static int __cpuinitdata multi_checked; -static int __cpuinitdata multi; +static int multi_checked; +static int multi; -static int __cpuinit set_multi(const struct dmi_system_id *d) +static int set_multi(const struct dmi_system_id *d) { if (multi) return 0; @@ -2424,7 +2424,7 @@ static int __cpuinit set_multi(const struct dmi_system_id *d) return 0; } -static const __cpuinitconst struct dmi_system_id multi_dmi_table[] = { +static const struct dmi_system_id multi_dmi_table[] = { { .callback = set_multi, .ident = "IBM System Summit2", @@ -2436,7 +2436,7 @@ static const __cpuinitconst struct dmi_system_id multi_dmi_table[] = { {} }; -static void __cpuinit dmi_check_multi(void) +static void dmi_check_multi(void) { if (multi_checked) return; @@ -2453,7 +2453,7 @@ static void __cpuinit dmi_check_multi(void) * multi-chassis. * Use DMI to check them */ -__cpuinit int apic_is_clustered_box(void) +int apic_is_clustered_box(void) { dmi_check_multi(); if (multi) diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index 9a9110918ca71..3e67f9e3d7ef1 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -74,7 +74,7 @@ static int numachip_phys_pkg_id(int initial_apic_id, int index_msb) return initial_apic_id >> index_msb; } -static int __cpuinit numachip_wakeup_secondary(int phys_apicid, unsigned long start_rip) +static int numachip_wakeup_secondary(int phys_apicid, unsigned long start_rip) { union numachip_csr_g3_ext_irq_gen int_gen; diff --git a/arch/x86/kernel/apic/es7000_32.c b/arch/x86/kernel/apic/es7000_32.c index 0874799a98c6f..c55224731b2d7 100644 --- a/arch/x86/kernel/apic/es7000_32.c +++ b/arch/x86/kernel/apic/es7000_32.c @@ -130,7 +130,7 @@ int es7000_plat; */ -static int __cpuinit wakeup_secondary_cpu_via_mip(int cpu, unsigned long eip) +static int wakeup_secondary_cpu_via_mip(int cpu, unsigned long eip) { unsigned long vect = 0, psaival = 0; diff --git a/arch/x86/kernel/apic/numaq_32.c b/arch/x86/kernel/apic/numaq_32.c index d661ee95cabfa..1e42e8f305ee2 100644 --- a/arch/x86/kernel/apic/numaq_32.c +++ b/arch/x86/kernel/apic/numaq_32.c @@ -105,7 +105,7 @@ static void __init smp_dump_qct(void) } } -void __cpuinit numaq_tsc_disable(void) +void numaq_tsc_disable(void) { if (!found_numaq) return; diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index c88baa4ff0e56..140e29db478d7 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -148,7 +148,7 @@ static void init_x2apic_ldr(void) /* * At CPU state changes, update the x2apic cluster sibling info. */ -static int __cpuinit +static int update_clusterinfo(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int this_cpu = (unsigned long)hcpu; diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index b32dbb411a9ae..578d821d75f25 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -194,7 +194,7 @@ EXPORT_SYMBOL_GPL(uv_possible_blades); unsigned long sn_rtc_cycles_per_second; EXPORT_SYMBOL(sn_rtc_cycles_per_second); -static int __cpuinit uv_wakeup_secondary(int phys_apicid, unsigned long start_rip) +static int uv_wakeup_secondary(int phys_apicid, unsigned long start_rip) { #ifdef CONFIG_SMP unsigned long val; @@ -401,7 +401,7 @@ static struct apic __refdata apic_x2apic_uv_x = { .safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle, }; -static __cpuinit void set_x2apic_extra_bits(int pnode) +static void set_x2apic_extra_bits(int pnode) { __this_cpu_write(x2apic_extra_bits, pnode << uvh_apicid.s.pnode_shift); } @@ -683,7 +683,7 @@ static void uv_heartbeat(unsigned long ignored) mod_timer_pinned(timer, jiffies + SCIR_CPU_HB_INTERVAL); } -static void __cpuinit uv_heartbeat_enable(int cpu) +static void uv_heartbeat_enable(int cpu) { while (!uv_cpu_hub_info(cpu)->scir.enabled) { struct timer_list *timer = &uv_cpu_hub_info(cpu)->scir.timer; @@ -700,7 +700,7 @@ static void __cpuinit uv_heartbeat_enable(int cpu) } #ifdef CONFIG_HOTPLUG_CPU -static void __cpuinit uv_heartbeat_disable(int cpu) +static void uv_heartbeat_disable(int cpu) { if (uv_cpu_hub_info(cpu)->scir.enabled) { uv_cpu_hub_info(cpu)->scir.enabled = 0; @@ -712,7 +712,7 @@ static void __cpuinit uv_heartbeat_disable(int cpu) /* * cpu hotplug notifier */ -static __cpuinit int uv_scir_cpu_notify(struct notifier_block *self, +static int uv_scir_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { long cpu = (long)hcpu; @@ -783,7 +783,7 @@ int uv_set_vga_state(struct pci_dev *pdev, bool decode, * Called on each cpu to initialize the per_cpu UV data area. * FIXME: hotplug not supported yet */ -void __cpuinit uv_cpu_init(void) +void uv_cpu_init(void) { /* CPU 0 initilization will be done via uv_system_init. */ if (!uv_blade_info) diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index ae177a014180b..bb4f57d5b7fd8 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -69,7 +69,7 @@ static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val) extern void vide(void); __asm__(".align 4\nvide: ret"); -static void __cpuinit init_amd_k5(struct cpuinfo_x86 *c) +static void init_amd_k5(struct cpuinfo_x86 *c) { /* * General Systems BIOSen alias the cpu frequency registers @@ -87,7 +87,7 @@ static void __cpuinit init_amd_k5(struct cpuinfo_x86 *c) } -static void __cpuinit init_amd_k6(struct cpuinfo_x86 *c) +static void init_amd_k6(struct cpuinfo_x86 *c) { u32 l, h; int mbytes = num_physpages >> (20-PAGE_SHIFT); @@ -179,7 +179,7 @@ static void __cpuinit init_amd_k6(struct cpuinfo_x86 *c) } } -static void __cpuinit amd_k7_smp_check(struct cpuinfo_x86 *c) +static void amd_k7_smp_check(struct cpuinfo_x86 *c) { /* calling is from identify_secondary_cpu() ? */ if (!c->cpu_index) @@ -222,7 +222,7 @@ static void __cpuinit amd_k7_smp_check(struct cpuinfo_x86 *c) add_taint(TAINT_UNSAFE_SMP, LOCKDEP_NOW_UNRELIABLE); } -static void __cpuinit init_amd_k7(struct cpuinfo_x86 *c) +static void init_amd_k7(struct cpuinfo_x86 *c) { u32 l, h; @@ -267,7 +267,7 @@ static void __cpuinit init_amd_k7(struct cpuinfo_x86 *c) * To workaround broken NUMA config. Read the comment in * srat_detect_node(). */ -static int __cpuinit nearby_node(int apicid) +static int nearby_node(int apicid) { int i, node; @@ -292,7 +292,7 @@ static int __cpuinit nearby_node(int apicid) * (2) AMD processors supporting compute units */ #ifdef CONFIG_X86_HT -static void __cpuinit amd_get_topology(struct cpuinfo_x86 *c) +static void amd_get_topology(struct cpuinfo_x86 *c) { u32 nodes, cores_per_cu = 1; u8 node_id; @@ -342,7 +342,7 @@ static void __cpuinit amd_get_topology(struct cpuinfo_x86 *c) * On a AMD dual core setup the lower bits of the APIC id distingush the cores. * Assumes number of cores is a power of two. */ -static void __cpuinit amd_detect_cmp(struct cpuinfo_x86 *c) +static void amd_detect_cmp(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_HT unsigned bits; @@ -369,7 +369,7 @@ u16 amd_get_nb_id(int cpu) } EXPORT_SYMBOL_GPL(amd_get_nb_id); -static void __cpuinit srat_detect_node(struct cpuinfo_x86 *c) +static void srat_detect_node(struct cpuinfo_x86 *c) { #ifdef CONFIG_NUMA int cpu = smp_processor_id(); @@ -421,7 +421,7 @@ static void __cpuinit srat_detect_node(struct cpuinfo_x86 *c) #endif } -static void __cpuinit early_init_amd_mc(struct cpuinfo_x86 *c) +static void early_init_amd_mc(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_HT unsigned bits, ecx; @@ -447,7 +447,7 @@ static void __cpuinit early_init_amd_mc(struct cpuinfo_x86 *c) #endif } -static void __cpuinit bsp_init_amd(struct cpuinfo_x86 *c) +static void bsp_init_amd(struct cpuinfo_x86 *c) { if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) { @@ -475,7 +475,7 @@ static void __cpuinit bsp_init_amd(struct cpuinfo_x86 *c) } } -static void __cpuinit early_init_amd(struct cpuinfo_x86 *c) +static void early_init_amd(struct cpuinfo_x86 *c) { early_init_amd_mc(c); @@ -524,7 +524,7 @@ static const int amd_erratum_383[]; static const int amd_erratum_400[]; static bool cpu_has_amd_erratum(const int *erratum); -static void __cpuinit init_amd(struct cpuinfo_x86 *c) +static void init_amd(struct cpuinfo_x86 *c) { u32 dummy; unsigned long long value; @@ -750,7 +750,7 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c) } #ifdef CONFIG_X86_32 -static unsigned int __cpuinit amd_size_cache(struct cpuinfo_x86 *c, +static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size) { /* AMD errata T13 (order #21922) */ @@ -767,7 +767,7 @@ static unsigned int __cpuinit amd_size_cache(struct cpuinfo_x86 *c, } #endif -static void __cpuinit cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c) +static void cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c) { tlb_flushall_shift = 5; @@ -775,7 +775,7 @@ static void __cpuinit cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c) tlb_flushall_shift = 4; } -static void __cpuinit cpu_detect_tlb_amd(struct cpuinfo_x86 *c) +static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c) { u32 ebx, eax, ecx, edx; u16 mask = 0xfff; @@ -830,7 +830,7 @@ static void __cpuinit cpu_detect_tlb_amd(struct cpuinfo_x86 *c) cpu_set_tlb_flushall_shift(c); } -static const struct cpu_dev __cpuinitconst amd_cpu_dev = { +static const struct cpu_dev amd_cpu_dev = { .c_vendor = "AMD", .c_ident = { "AuthenticAMD" }, #ifdef CONFIG_X86_32 diff --git a/arch/x86/kernel/cpu/centaur.c b/arch/x86/kernel/cpu/centaur.c index 159103c0b1f4b..fbf6c3bc2400a 100644 --- a/arch/x86/kernel/cpu/centaur.c +++ b/arch/x86/kernel/cpu/centaur.c @@ -11,7 +11,7 @@ #ifdef CONFIG_X86_OOSTORE -static u32 __cpuinit power2(u32 x) +static u32 power2(u32 x) { u32 s = 1; @@ -25,7 +25,7 @@ static u32 __cpuinit power2(u32 x) /* * Set up an actual MCR */ -static void __cpuinit centaur_mcr_insert(int reg, u32 base, u32 size, int key) +static void centaur_mcr_insert(int reg, u32 base, u32 size, int key) { u32 lo, hi; @@ -42,7 +42,7 @@ static void __cpuinit centaur_mcr_insert(int reg, u32 base, u32 size, int key) * * Shortcut: We know you can't put 4Gig of RAM on a winchip */ -static u32 __cpuinit ramtop(void) +static u32 ramtop(void) { u32 clip = 0xFFFFFFFFUL; u32 top = 0; @@ -91,7 +91,7 @@ static u32 __cpuinit ramtop(void) /* * Compute a set of MCR's to give maximum coverage */ -static int __cpuinit centaur_mcr_compute(int nr, int key) +static int centaur_mcr_compute(int nr, int key) { u32 mem = ramtop(); u32 root = power2(mem); @@ -157,7 +157,7 @@ static int __cpuinit centaur_mcr_compute(int nr, int key) return ct; } -static void __cpuinit centaur_create_optimal_mcr(void) +static void centaur_create_optimal_mcr(void) { int used; int i; @@ -181,7 +181,7 @@ static void __cpuinit centaur_create_optimal_mcr(void) wrmsr(MSR_IDT_MCR0+i, 0, 0); } -static void __cpuinit winchip2_create_optimal_mcr(void) +static void winchip2_create_optimal_mcr(void) { u32 lo, hi; int used; @@ -217,7 +217,7 @@ static void __cpuinit winchip2_create_optimal_mcr(void) /* * Handle the MCR key on the Winchip 2. */ -static void __cpuinit winchip2_unprotect_mcr(void) +static void winchip2_unprotect_mcr(void) { u32 lo, hi; u32 key; @@ -229,7 +229,7 @@ static void __cpuinit winchip2_unprotect_mcr(void) wrmsr(MSR_IDT_MCR_CTRL, lo, hi); } -static void __cpuinit winchip2_protect_mcr(void) +static void winchip2_protect_mcr(void) { u32 lo, hi; @@ -247,7 +247,7 @@ static void __cpuinit winchip2_protect_mcr(void) #define RNG_ENABLED (1 << 3) #define RNG_ENABLE (1 << 6) /* MSR_VIA_RNG */ -static void __cpuinit init_c3(struct cpuinfo_x86 *c) +static void init_c3(struct cpuinfo_x86 *c) { u32 lo, hi; @@ -318,7 +318,7 @@ enum { EAMD3D = 1<<20, }; -static void __cpuinit early_init_centaur(struct cpuinfo_x86 *c) +static void early_init_centaur(struct cpuinfo_x86 *c) { switch (c->x86) { #ifdef CONFIG_X86_32 @@ -337,7 +337,7 @@ static void __cpuinit early_init_centaur(struct cpuinfo_x86 *c) #endif } -static void __cpuinit init_centaur(struct cpuinfo_x86 *c) +static void init_centaur(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_32 char *name; @@ -468,7 +468,7 @@ static void __cpuinit init_centaur(struct cpuinfo_x86 *c) #endif } -static unsigned int __cpuinit +static unsigned int centaur_size_cache(struct cpuinfo_x86 *c, unsigned int size) { #ifdef CONFIG_X86_32 @@ -488,7 +488,7 @@ centaur_size_cache(struct cpuinfo_x86 *c, unsigned int size) return size; } -static const struct cpu_dev __cpuinitconst centaur_cpu_dev = { +static const struct cpu_dev centaur_cpu_dev = { .c_vendor = "Centaur", .c_ident = { "CentaurHauls" }, .c_early_init = early_init_centaur, diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 7e09789d2cf38..6a9c29e1a8c50 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -63,7 +63,7 @@ void __init setup_cpu_local_masks(void) alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask); } -static void __cpuinit default_init(struct cpuinfo_x86 *c) +static void default_init(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_64 cpu_detect_cache_sizes(c); @@ -80,13 +80,13 @@ static void __cpuinit default_init(struct cpuinfo_x86 *c) #endif } -static const struct cpu_dev __cpuinitconst default_cpu = { +static const struct cpu_dev default_cpu = { .c_init = default_init, .c_vendor = "Unknown", .c_x86_vendor = X86_VENDOR_UNKNOWN, }; -static const struct cpu_dev *this_cpu __cpuinitdata = &default_cpu; +static const struct cpu_dev *this_cpu = &default_cpu; DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = { #ifdef CONFIG_X86_64 @@ -162,8 +162,8 @@ static int __init x86_xsaveopt_setup(char *s) __setup("noxsaveopt", x86_xsaveopt_setup); #ifdef CONFIG_X86_32 -static int cachesize_override __cpuinitdata = -1; -static int disable_x86_serial_nr __cpuinitdata = 1; +static int cachesize_override = -1; +static int disable_x86_serial_nr = 1; static int __init cachesize_setup(char *str) { @@ -217,12 +217,12 @@ static inline int flag_is_changeable_p(u32 flag) } /* Probe for the CPUID instruction */ -int __cpuinit have_cpuid_p(void) +int have_cpuid_p(void) { return flag_is_changeable_p(X86_EFLAGS_ID); } -static void __cpuinit squash_the_stupid_serial_number(struct cpuinfo_x86 *c) +static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c) { unsigned long lo, hi; @@ -312,7 +312,7 @@ cpuid_dependent_features[] = { { 0, 0 } }; -static void __cpuinit filter_cpuid_features(struct cpuinfo_x86 *c, bool warn) +static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn) { const struct cpuid_dependent_feature *df; @@ -350,7 +350,7 @@ static void __cpuinit filter_cpuid_features(struct cpuinfo_x86 *c, bool warn) */ /* Look up CPU names by table lookup. */ -static const char *__cpuinit table_lookup_model(struct cpuinfo_x86 *c) +static const char *table_lookup_model(struct cpuinfo_x86 *c) { const struct cpu_model_info *info; @@ -400,9 +400,9 @@ void switch_to_new_gdt(int cpu) load_percpu_segment(cpu); } -static const struct cpu_dev *__cpuinitdata cpu_devs[X86_VENDOR_NUM] = {}; +static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {}; -static void __cpuinit get_model_name(struct cpuinfo_x86 *c) +static void get_model_name(struct cpuinfo_x86 *c) { unsigned int *v; char *p, *q; @@ -431,7 +431,7 @@ static void __cpuinit get_model_name(struct cpuinfo_x86 *c) } } -void __cpuinit cpu_detect_cache_sizes(struct cpuinfo_x86 *c) +void cpu_detect_cache_sizes(struct cpuinfo_x86 *c) { unsigned int n, dummy, ebx, ecx, edx, l2size; @@ -485,7 +485,7 @@ u16 __read_mostly tlb_lld_4m[NR_INFO]; */ s8 __read_mostly tlb_flushall_shift = -1; -void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c) +void cpu_detect_tlb(struct cpuinfo_x86 *c) { if (this_cpu->c_detect_tlb) this_cpu->c_detect_tlb(c); @@ -499,7 +499,7 @@ void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c) tlb_flushall_shift); } -void __cpuinit detect_ht(struct cpuinfo_x86 *c) +void detect_ht(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_HT u32 eax, ebx, ecx, edx; @@ -550,7 +550,7 @@ void __cpuinit detect_ht(struct cpuinfo_x86 *c) #endif } -static void __cpuinit get_cpu_vendor(struct cpuinfo_x86 *c) +static void get_cpu_vendor(struct cpuinfo_x86 *c) { char *v = c->x86_vendor_id; int i; @@ -577,7 +577,7 @@ static void __cpuinit get_cpu_vendor(struct cpuinfo_x86 *c) this_cpu = &default_cpu; } -void __cpuinit cpu_detect(struct cpuinfo_x86 *c) +void cpu_detect(struct cpuinfo_x86 *c) { /* Get vendor name */ cpuid(0x00000000, (unsigned int *)&c->cpuid_level, @@ -607,7 +607,7 @@ void __cpuinit cpu_detect(struct cpuinfo_x86 *c) } } -void __cpuinit get_cpu_cap(struct cpuinfo_x86 *c) +void get_cpu_cap(struct cpuinfo_x86 *c) { u32 tfms, xlvl; u32 ebx; @@ -658,7 +658,7 @@ void __cpuinit get_cpu_cap(struct cpuinfo_x86 *c) init_scattered_cpuid_features(c); } -static void __cpuinit identify_cpu_without_cpuid(struct cpuinfo_x86 *c) +static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_32 int i; @@ -774,7 +774,7 @@ void __init early_cpu_init(void) * unless we can find a reliable way to detect all the broken cases. * Enable it explicitly on 64-bit for non-constant inputs of cpu_has(). */ -static void __cpuinit detect_nopl(struct cpuinfo_x86 *c) +static void detect_nopl(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_32 clear_cpu_cap(c, X86_FEATURE_NOPL); @@ -783,7 +783,7 @@ static void __cpuinit detect_nopl(struct cpuinfo_x86 *c) #endif } -static void __cpuinit generic_identify(struct cpuinfo_x86 *c) +static void generic_identify(struct cpuinfo_x86 *c) { c->extended_cpuid_level = 0; @@ -820,7 +820,7 @@ static void __cpuinit generic_identify(struct cpuinfo_x86 *c) /* * This does the hard work of actually picking apart the CPU stuff... */ -static void __cpuinit identify_cpu(struct cpuinfo_x86 *c) +static void identify_cpu(struct cpuinfo_x86 *c) { int i; @@ -965,7 +965,7 @@ void __init identify_boot_cpu(void) cpu_detect_tlb(&boot_cpu_data); } -void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c) +void identify_secondary_cpu(struct cpuinfo_x86 *c) { BUG_ON(c == &boot_cpu_data); identify_cpu(c); @@ -980,14 +980,14 @@ struct msr_range { unsigned max; }; -static const struct msr_range msr_range_array[] __cpuinitconst = { +static const struct msr_range msr_range_array[] = { { 0x00000000, 0x00000418}, { 0xc0000000, 0xc000040b}, { 0xc0010000, 0xc0010142}, { 0xc0011000, 0xc001103b}, }; -static void __cpuinit __print_cpu_msr(void) +static void __print_cpu_msr(void) { unsigned index_min, index_max; unsigned index; @@ -1027,7 +1027,7 @@ static __init int setup_noclflush(char *arg) } __setup("noclflush", setup_noclflush); -void __cpuinit print_cpu_info(struct cpuinfo_x86 *c) +void print_cpu_info(struct cpuinfo_x86 *c) { const char *vendor = NULL; @@ -1056,7 +1056,7 @@ void __cpuinit print_cpu_info(struct cpuinfo_x86 *c) print_cpu_msr(c); } -void __cpuinit print_cpu_msr(struct cpuinfo_x86 *c) +void print_cpu_msr(struct cpuinfo_x86 *c) { if (c->cpu_index < show_msr) __print_cpu_msr(); @@ -1221,7 +1221,7 @@ static void dbg_restore_debug_regs(void) */ #ifdef CONFIG_X86_64 -void __cpuinit cpu_init(void) +void cpu_init(void) { struct orig_ist *oist; struct task_struct *me; @@ -1320,7 +1320,7 @@ void __cpuinit cpu_init(void) #else -void __cpuinit cpu_init(void) +void cpu_init(void) { int cpu = smp_processor_id(); struct task_struct *curr = current; diff --git a/arch/x86/kernel/cpu/cyrix.c b/arch/x86/kernel/cpu/cyrix.c index d048d5ca43c1c..f1fa2f2c2c906 100644 --- a/arch/x86/kernel/cpu/cyrix.c +++ b/arch/x86/kernel/cpu/cyrix.c @@ -15,7 +15,7 @@ /* * Read NSC/Cyrix DEVID registers (DIR) to get more detailed info. about the CPU */ -static void __cpuinit __do_cyrix_devid(unsigned char *dir0, unsigned char *dir1) +static void __do_cyrix_devid(unsigned char *dir0, unsigned char *dir1) { unsigned char ccr2, ccr3; @@ -44,7 +44,7 @@ static void __cpuinit __do_cyrix_devid(unsigned char *dir0, unsigned char *dir1) } } -static void __cpuinit do_cyrix_devid(unsigned char *dir0, unsigned char *dir1) +static void do_cyrix_devid(unsigned char *dir0, unsigned char *dir1) { unsigned long flags; @@ -59,25 +59,25 @@ static void __cpuinit do_cyrix_devid(unsigned char *dir0, unsigned char *dir1) * Actually since bugs.h doesn't even reference this perhaps someone should * fix the documentation ??? */ -static unsigned char Cx86_dir0_msb __cpuinitdata = 0; +static unsigned char Cx86_dir0_msb = 0; -static const char __cpuinitconst Cx86_model[][9] = { +static const char Cx86_model[][9] = { "Cx486", "Cx486", "5x86 ", "6x86", "MediaGX ", "6x86MX ", "M II ", "Unknown" }; -static const char __cpuinitconst Cx486_name[][5] = { +static const char Cx486_name[][5] = { "SLC", "DLC", "SLC2", "DLC2", "SRx", "DRx", "SRx2", "DRx2" }; -static const char __cpuinitconst Cx486S_name[][4] = { +static const char Cx486S_name[][4] = { "S", "S2", "Se", "S2e" }; -static const char __cpuinitconst Cx486D_name[][4] = { +static const char Cx486D_name[][4] = { "DX", "DX2", "?", "?", "?", "DX4" }; -static char Cx86_cb[] __cpuinitdata = "?.5x Core/Bus Clock"; -static const char __cpuinitconst cyrix_model_mult1[] = "12??43"; -static const char __cpuinitconst cyrix_model_mult2[] = "12233445"; +static char Cx86_cb[] = "?.5x Core/Bus Clock"; +static const char cyrix_model_mult1[] = "12??43"; +static const char cyrix_model_mult2[] = "12233445"; /* * Reset the slow-loop (SLOP) bit on the 686(L) which is set by some old @@ -87,7 +87,7 @@ static const char __cpuinitconst cyrix_model_mult2[] = "12233445"; * FIXME: our newer udelay uses the tsc. We don't need to frob with SLOP */ -static void __cpuinit check_cx686_slop(struct cpuinfo_x86 *c) +static void check_cx686_slop(struct cpuinfo_x86 *c) { unsigned long flags; @@ -112,7 +112,7 @@ static void __cpuinit check_cx686_slop(struct cpuinfo_x86 *c) } -static void __cpuinit set_cx86_reorder(void) +static void set_cx86_reorder(void) { u8 ccr3; @@ -127,7 +127,7 @@ static void __cpuinit set_cx86_reorder(void) setCx86(CX86_CCR3, ccr3); } -static void __cpuinit set_cx86_memwb(void) +static void set_cx86_memwb(void) { printk(KERN_INFO "Enable Memory-Write-back mode on Cyrix/NSC processor.\n"); @@ -143,7 +143,7 @@ static void __cpuinit set_cx86_memwb(void) * Configure later MediaGX and/or Geode processor. */ -static void __cpuinit geode_configure(void) +static void geode_configure(void) { unsigned long flags; u8 ccr3; @@ -166,7 +166,7 @@ static void __cpuinit geode_configure(void) local_irq_restore(flags); } -static void __cpuinit early_init_cyrix(struct cpuinfo_x86 *c) +static void early_init_cyrix(struct cpuinfo_x86 *c) { unsigned char dir0, dir0_msn, dir1 = 0; @@ -185,7 +185,7 @@ static void __cpuinit early_init_cyrix(struct cpuinfo_x86 *c) } } -static void __cpuinit init_cyrix(struct cpuinfo_x86 *c) +static void init_cyrix(struct cpuinfo_x86 *c) { unsigned char dir0, dir0_msn, dir0_lsn, dir1 = 0; char *buf = c->x86_model_id; @@ -356,7 +356,7 @@ static void __cpuinit init_cyrix(struct cpuinfo_x86 *c) /* * Handle National Semiconductor branded processors */ -static void __cpuinit init_nsc(struct cpuinfo_x86 *c) +static void init_nsc(struct cpuinfo_x86 *c) { /* * There may be GX1 processors in the wild that are branded @@ -405,7 +405,7 @@ static inline int test_cyrix_52div(void) return (unsigned char) (test >> 8) == 0x02; } -static void __cpuinit cyrix_identify(struct cpuinfo_x86 *c) +static void cyrix_identify(struct cpuinfo_x86 *c) { /* Detect Cyrix with disabled CPUID */ if (c->x86 == 4 && test_cyrix_52div()) { @@ -441,7 +441,7 @@ static void __cpuinit cyrix_identify(struct cpuinfo_x86 *c) } } -static const struct cpu_dev __cpuinitconst cyrix_cpu_dev = { +static const struct cpu_dev cyrix_cpu_dev = { .c_vendor = "Cyrix", .c_ident = { "CyrixInstead" }, .c_early_init = early_init_cyrix, @@ -452,7 +452,7 @@ static const struct cpu_dev __cpuinitconst cyrix_cpu_dev = { cpu_dev_register(cyrix_cpu_dev); -static const struct cpu_dev __cpuinitconst nsc_cpu_dev = { +static const struct cpu_dev nsc_cpu_dev = { .c_vendor = "NSC", .c_ident = { "Geode by NSC" }, .c_init = init_nsc, diff --git a/arch/x86/kernel/cpu/hypervisor.c b/arch/x86/kernel/cpu/hypervisor.c index 1e7e84a02ebaf..87279212d3186 100644 --- a/arch/x86/kernel/cpu/hypervisor.c +++ b/arch/x86/kernel/cpu/hypervisor.c @@ -60,7 +60,7 @@ detect_hypervisor_vendor(void) } } -void __cpuinit init_hypervisor(struct cpuinfo_x86 *c) +void init_hypervisor(struct cpuinfo_x86 *c) { if (x86_hyper && x86_hyper->set_cpu_features) x86_hyper->set_cpu_features(c); diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index 8533e69d2b89f..1d8152b764a7c 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -26,7 +26,7 @@ #include #endif -static void __cpuinit early_init_intel(struct cpuinfo_x86 *c) +static void early_init_intel(struct cpuinfo_x86 *c) { u64 misc_enable; @@ -178,7 +178,7 @@ static void __cpuinit early_init_intel(struct cpuinfo_x86 *c) * This is called before we do cpu ident work */ -int __cpuinit ppro_with_ram_bug(void) +int ppro_with_ram_bug(void) { /* Uses data from early_cpu_detect now */ if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && @@ -191,7 +191,7 @@ int __cpuinit ppro_with_ram_bug(void) return 0; } -static void __cpuinit intel_smp_check(struct cpuinfo_x86 *c) +static void intel_smp_check(struct cpuinfo_x86 *c) { /* calling is from identify_secondary_cpu() ? */ if (!c->cpu_index) @@ -211,7 +211,7 @@ static void __cpuinit intel_smp_check(struct cpuinfo_x86 *c) } } -static void __cpuinit intel_workarounds(struct cpuinfo_x86 *c) +static void intel_workarounds(struct cpuinfo_x86 *c) { unsigned long lo, hi; @@ -290,12 +290,12 @@ static void __cpuinit intel_workarounds(struct cpuinfo_x86 *c) intel_smp_check(c); } #else -static void __cpuinit intel_workarounds(struct cpuinfo_x86 *c) +static void intel_workarounds(struct cpuinfo_x86 *c) { } #endif -static void __cpuinit srat_detect_node(struct cpuinfo_x86 *c) +static void srat_detect_node(struct cpuinfo_x86 *c) { #ifdef CONFIG_NUMA unsigned node; @@ -315,7 +315,7 @@ static void __cpuinit srat_detect_node(struct cpuinfo_x86 *c) /* * find out the number of processor cores on the die */ -static int __cpuinit intel_num_cpu_cores(struct cpuinfo_x86 *c) +static int intel_num_cpu_cores(struct cpuinfo_x86 *c) { unsigned int eax, ebx, ecx, edx; @@ -330,7 +330,7 @@ static int __cpuinit intel_num_cpu_cores(struct cpuinfo_x86 *c) return 1; } -static void __cpuinit detect_vmx_virtcap(struct cpuinfo_x86 *c) +static void detect_vmx_virtcap(struct cpuinfo_x86 *c) { /* Intel VMX MSR indicated features */ #define X86_VMX_FEATURE_PROC_CTLS_TPR_SHADOW 0x00200000 @@ -368,7 +368,7 @@ static void __cpuinit detect_vmx_virtcap(struct cpuinfo_x86 *c) } } -static void __cpuinit init_intel(struct cpuinfo_x86 *c) +static void init_intel(struct cpuinfo_x86 *c) { unsigned int l2 = 0; @@ -488,7 +488,7 @@ static void __cpuinit init_intel(struct cpuinfo_x86 *c) } #ifdef CONFIG_X86_32 -static unsigned int __cpuinit intel_size_cache(struct cpuinfo_x86 *c, unsigned int size) +static unsigned int intel_size_cache(struct cpuinfo_x86 *c, unsigned int size) { /* * Intel PIII Tualatin. This comes in two flavours. @@ -522,7 +522,7 @@ static unsigned int __cpuinit intel_size_cache(struct cpuinfo_x86 *c, unsigned i #define STLB_4K 0x41 -static const struct _tlb_table intel_tlb_table[] __cpuinitconst = { +static const struct _tlb_table intel_tlb_table[] = { { 0x01, TLB_INST_4K, 32, " TLB_INST 4 KByte pages, 4-way set associative" }, { 0x02, TLB_INST_4M, 2, " TLB_INST 4 MByte pages, full associative" }, { 0x03, TLB_DATA_4K, 64, " TLB_DATA 4 KByte pages, 4-way set associative" }, @@ -552,7 +552,7 @@ static const struct _tlb_table intel_tlb_table[] __cpuinitconst = { { 0x00, 0, 0 } }; -static void __cpuinit intel_tlb_lookup(const unsigned char desc) +static void intel_tlb_lookup(const unsigned char desc) { unsigned char k; if (desc == 0) @@ -621,7 +621,7 @@ static void __cpuinit intel_tlb_lookup(const unsigned char desc) } } -static void __cpuinit intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c) +static void intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c) { switch ((c->x86 << 8) + c->x86_model) { case 0x60f: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */ @@ -650,7 +650,7 @@ static void __cpuinit intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c) } } -static void __cpuinit intel_detect_tlb(struct cpuinfo_x86 *c) +static void intel_detect_tlb(struct cpuinfo_x86 *c) { int i, j, n; unsigned int regs[4]; @@ -677,7 +677,7 @@ static void __cpuinit intel_detect_tlb(struct cpuinfo_x86 *c) intel_tlb_flushall_shift_set(c); } -static const struct cpu_dev __cpuinitconst intel_cpu_dev = { +static const struct cpu_dev intel_cpu_dev = { .c_vendor = "Intel", .c_ident = { "GenuineIntel" }, #ifdef CONFIG_X86_32 diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c index 7c6f7d548c0f9..3ef4c94a2f5eb 100644 --- a/arch/x86/kernel/cpu/intel_cacheinfo.c +++ b/arch/x86/kernel/cpu/intel_cacheinfo.c @@ -37,7 +37,7 @@ struct _cache_table { /* All the cache descriptor types we care about (no TLB or trace cache entries) */ -static const struct _cache_table __cpuinitconst cache_table[] = +static const struct _cache_table cache_table[] = { { 0x06, LVL_1_INST, 8 }, /* 4-way set assoc, 32 byte line size */ { 0x08, LVL_1_INST, 16 }, /* 4-way set assoc, 32 byte line size */ @@ -203,7 +203,7 @@ union l3_cache { unsigned val; }; -static const unsigned short __cpuinitconst assocs[] = { +static const unsigned short assocs[] = { [1] = 1, [2] = 2, [4] = 4, @@ -217,10 +217,10 @@ static const unsigned short __cpuinitconst assocs[] = { [0xf] = 0xffff /* fully associative - no way to show this currently */ }; -static const unsigned char __cpuinitconst levels[] = { 1, 1, 2, 3 }; -static const unsigned char __cpuinitconst types[] = { 1, 2, 3, 3 }; +static const unsigned char levels[] = { 1, 1, 2, 3 }; +static const unsigned char types[] = { 1, 2, 3, 3 }; -static void __cpuinit +static void amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax, union _cpuid4_leaf_ebx *ebx, union _cpuid4_leaf_ecx *ecx) @@ -302,7 +302,7 @@ struct _cache_attr { /* * L3 cache descriptors */ -static void __cpuinit amd_calc_l3_indices(struct amd_northbridge *nb) +static void amd_calc_l3_indices(struct amd_northbridge *nb) { struct amd_l3_cache *l3 = &nb->l3_cache; unsigned int sc0, sc1, sc2, sc3; @@ -325,7 +325,7 @@ static void __cpuinit amd_calc_l3_indices(struct amd_northbridge *nb) l3->indices = (max(max3(sc0, sc1, sc2), sc3) << 10) - 1; } -static void __cpuinit amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf, int index) +static void amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf, int index) { int node; @@ -528,7 +528,7 @@ static struct _cache_attr subcaches = #endif /* CONFIG_AMD_NB && CONFIG_SYSFS */ static int -__cpuinit cpuid4_cache_lookup_regs(int index, +cpuid4_cache_lookup_regs(int index, struct _cpuid4_info_regs *this_leaf) { union _cpuid4_leaf_eax eax; @@ -560,7 +560,7 @@ __cpuinit cpuid4_cache_lookup_regs(int index, return 0; } -static int __cpuinit find_num_cache_leaves(struct cpuinfo_x86 *c) +static int find_num_cache_leaves(struct cpuinfo_x86 *c) { unsigned int eax, ebx, ecx, edx, op; union _cpuid4_leaf_eax cache_eax; @@ -580,7 +580,7 @@ static int __cpuinit find_num_cache_leaves(struct cpuinfo_x86 *c) return i; } -void __cpuinit init_amd_cacheinfo(struct cpuinfo_x86 *c) +void init_amd_cacheinfo(struct cpuinfo_x86 *c) { if (cpu_has_topoext) { @@ -593,7 +593,7 @@ void __cpuinit init_amd_cacheinfo(struct cpuinfo_x86 *c) } } -unsigned int __cpuinit init_intel_cacheinfo(struct cpuinfo_x86 *c) +unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c) { /* Cache sizes */ unsigned int trace = 0, l1i = 0, l1d = 0, l2 = 0, l3 = 0; @@ -746,7 +746,7 @@ static DEFINE_PER_CPU(struct _cpuid4_info *, ici_cpuid4_info); #ifdef CONFIG_SMP -static int __cpuinit cache_shared_amd_cpu_map_setup(unsigned int cpu, int index) +static int cache_shared_amd_cpu_map_setup(unsigned int cpu, int index) { struct _cpuid4_info *this_leaf; int i, sibling; @@ -795,7 +795,7 @@ static int __cpuinit cache_shared_amd_cpu_map_setup(unsigned int cpu, int index) return 1; } -static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index) +static void cache_shared_cpu_map_setup(unsigned int cpu, int index) { struct _cpuid4_info *this_leaf, *sibling_leaf; unsigned long num_threads_sharing; @@ -830,7 +830,7 @@ static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index) } } } -static void __cpuinit cache_remove_shared_cpu_map(unsigned int cpu, int index) +static void cache_remove_shared_cpu_map(unsigned int cpu, int index) { struct _cpuid4_info *this_leaf, *sibling_leaf; int sibling; @@ -843,16 +843,16 @@ static void __cpuinit cache_remove_shared_cpu_map(unsigned int cpu, int index) } } #else -static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index) +static void cache_shared_cpu_map_setup(unsigned int cpu, int index) { } -static void __cpuinit cache_remove_shared_cpu_map(unsigned int cpu, int index) +static void cache_remove_shared_cpu_map(unsigned int cpu, int index) { } #endif -static void __cpuinit free_cache_attributes(unsigned int cpu) +static void free_cache_attributes(unsigned int cpu) { int i; @@ -863,7 +863,7 @@ static void __cpuinit free_cache_attributes(unsigned int cpu) per_cpu(ici_cpuid4_info, cpu) = NULL; } -static void __cpuinit get_cpu_leaves(void *_retval) +static void get_cpu_leaves(void *_retval) { int j, *retval = _retval, cpu = smp_processor_id(); @@ -883,7 +883,7 @@ static void __cpuinit get_cpu_leaves(void *_retval) } } -static int __cpuinit detect_cache_attributes(unsigned int cpu) +static int detect_cache_attributes(unsigned int cpu) { int retval; @@ -1017,7 +1017,7 @@ static struct attribute *default_attrs[] = { }; #ifdef CONFIG_AMD_NB -static struct attribute ** __cpuinit amd_l3_attrs(void) +static struct attribute ** amd_l3_attrs(void) { static struct attribute **attrs; int n; @@ -1093,7 +1093,7 @@ static struct kobj_type ktype_percpu_entry = { .sysfs_ops = &sysfs_ops, }; -static void __cpuinit cpuid4_cache_sysfs_exit(unsigned int cpu) +static void cpuid4_cache_sysfs_exit(unsigned int cpu) { kfree(per_cpu(ici_cache_kobject, cpu)); kfree(per_cpu(ici_index_kobject, cpu)); @@ -1102,7 +1102,7 @@ static void __cpuinit cpuid4_cache_sysfs_exit(unsigned int cpu) free_cache_attributes(cpu); } -static int __cpuinit cpuid4_cache_sysfs_init(unsigned int cpu) +static int cpuid4_cache_sysfs_init(unsigned int cpu) { int err; @@ -1134,7 +1134,7 @@ static int __cpuinit cpuid4_cache_sysfs_init(unsigned int cpu) static DECLARE_BITMAP(cache_dev_map, NR_CPUS); /* Add/Remove cache interface for CPU device */ -static int __cpuinit cache_add_dev(struct device *dev) +static int cache_add_dev(struct device *dev) { unsigned int cpu = dev->id; unsigned long i, j; @@ -1185,7 +1185,7 @@ static int __cpuinit cache_add_dev(struct device *dev) return 0; } -static void __cpuinit cache_remove_dev(struct device *dev) +static void cache_remove_dev(struct device *dev) { unsigned int cpu = dev->id; unsigned long i; @@ -1202,7 +1202,7 @@ static void __cpuinit cache_remove_dev(struct device *dev) cpuid4_cache_sysfs_exit(cpu); } -static int __cpuinit cacheinfo_cpu_callback(struct notifier_block *nfb, +static int cacheinfo_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; @@ -1222,7 +1222,7 @@ static int __cpuinit cacheinfo_cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata cacheinfo_cpu_notifier = { +static struct notifier_block cacheinfo_cpu_notifier = { .notifier_call = cacheinfo_cpu_callback, }; diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index 9239504b41cbc..a33bca5083fb7 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -1360,7 +1360,7 @@ int mce_notify_irq(void) } EXPORT_SYMBOL_GPL(mce_notify_irq); -static int __cpuinit __mcheck_cpu_mce_banks_init(void) +static int __mcheck_cpu_mce_banks_init(void) { int i; u8 num_banks = mca_cfg.banks; @@ -1381,7 +1381,7 @@ static int __cpuinit __mcheck_cpu_mce_banks_init(void) /* * Initialize Machine Checks for a CPU. */ -static int __cpuinit __mcheck_cpu_cap_init(void) +static int __mcheck_cpu_cap_init(void) { unsigned b; u64 cap; @@ -1480,7 +1480,7 @@ static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs) } /* Add per CPU specific workarounds here */ -static int __cpuinit __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c) +static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c) { struct mca_config *cfg = &mca_cfg; @@ -1590,7 +1590,7 @@ static int __cpuinit __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c) return 0; } -static int __cpuinit __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c) +static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c) { if (c->x86 != 5) return 0; @@ -1661,7 +1661,7 @@ void (*machine_check_vector)(struct pt_regs *, long error_code) = * Called for each booted CPU to set up machine checks. * Must be called with preempt off: */ -void __cpuinit mcheck_cpu_init(struct cpuinfo_x86 *c) +void mcheck_cpu_init(struct cpuinfo_x86 *c) { if (mca_cfg.disabled) return; @@ -2225,7 +2225,7 @@ static void mce_device_release(struct device *dev) } /* Per cpu device init. All of the cpus still share the same ctrl bank: */ -static __cpuinit int mce_device_create(unsigned int cpu) +static int mce_device_create(unsigned int cpu) { struct device *dev; int err; @@ -2271,7 +2271,7 @@ static __cpuinit int mce_device_create(unsigned int cpu) return err; } -static __cpuinit void mce_device_remove(unsigned int cpu) +static void mce_device_remove(unsigned int cpu) { struct device *dev = per_cpu(mce_device, cpu); int i; @@ -2291,7 +2291,7 @@ static __cpuinit void mce_device_remove(unsigned int cpu) } /* Make sure there are no machine checks on offlined CPUs. */ -static void __cpuinit mce_disable_cpu(void *h) +static void mce_disable_cpu(void *h) { unsigned long action = *(unsigned long *)h; int i; @@ -2309,7 +2309,7 @@ static void __cpuinit mce_disable_cpu(void *h) } } -static void __cpuinit mce_reenable_cpu(void *h) +static void mce_reenable_cpu(void *h) { unsigned long action = *(unsigned long *)h; int i; @@ -2328,7 +2328,7 @@ static void __cpuinit mce_reenable_cpu(void *h) } /* Get notified when a cpu comes on/off. Be hotplug friendly. */ -static int __cpuinit +static int mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; @@ -2364,7 +2364,7 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) return NOTIFY_OK; } -static struct notifier_block mce_cpu_notifier __cpuinitdata = { +static struct notifier_block mce_cpu_notifier = { .notifier_call = mce_cpu_callback, }; diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c index 338a4ae486bc8..1246fa656739a 100644 --- a/arch/x86/kernel/cpu/mcheck/mce_amd.c +++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c @@ -458,7 +458,7 @@ static struct kobj_type threshold_ktype = { .default_attrs = default_attrs, }; -static __cpuinit int allocate_threshold_blocks(unsigned int cpu, +static int allocate_threshold_blocks(unsigned int cpu, unsigned int bank, unsigned int block, u32 address) @@ -543,7 +543,7 @@ static __cpuinit int allocate_threshold_blocks(unsigned int cpu, return err; } -static __cpuinit int __threshold_add_blocks(struct threshold_bank *b) +static int __threshold_add_blocks(struct threshold_bank *b) { struct list_head *head = &b->blocks->miscj; struct threshold_block *pos = NULL; @@ -567,7 +567,7 @@ static __cpuinit int __threshold_add_blocks(struct threshold_bank *b) return err; } -static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank) +static int threshold_create_bank(unsigned int cpu, unsigned int bank) { struct device *dev = per_cpu(mce_device, cpu); struct amd_northbridge *nb = NULL; @@ -632,7 +632,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank) } /* create dir/files for all valid threshold banks */ -static __cpuinit int threshold_create_device(unsigned int cpu) +static int threshold_create_device(unsigned int cpu) { unsigned int bank; struct threshold_bank **bp; @@ -736,7 +736,7 @@ static void threshold_remove_device(unsigned int cpu) } /* get notified when a cpu comes on/off */ -static void __cpuinit +static void amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu) { switch (action) { diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c index 47a1870279aad..7b7e8fdd771cb 100644 --- a/arch/x86/kernel/cpu/mcheck/therm_throt.c +++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c @@ -221,7 +221,7 @@ static int thresh_event_valid(int event) #ifdef CONFIG_SYSFS /* Add/Remove thermal_throttle interface for CPU device: */ -static __cpuinit int thermal_throttle_add_dev(struct device *dev, +static int thermal_throttle_add_dev(struct device *dev, unsigned int cpu) { int err; @@ -248,7 +248,7 @@ static __cpuinit int thermal_throttle_add_dev(struct device *dev, return err; } -static __cpuinit void thermal_throttle_remove_dev(struct device *dev) +static void thermal_throttle_remove_dev(struct device *dev) { sysfs_remove_group(&dev->kobj, &thermal_attr_group); } @@ -257,7 +257,7 @@ static __cpuinit void thermal_throttle_remove_dev(struct device *dev) static DEFINE_MUTEX(therm_cpu_lock); /* Get notified when a cpu comes on/off. Be hotplug friendly. */ -static __cpuinit int +static int thermal_throttle_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) @@ -288,7 +288,7 @@ thermal_throttle_cpu_callback(struct notifier_block *nfb, return notifier_from_errno(err); } -static struct notifier_block thermal_throttle_cpu_notifier __cpuinitdata = +static struct notifier_block thermal_throttle_cpu_notifier = { .notifier_call = thermal_throttle_cpu_callback, }; diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index 123d9e2271dc5..715528dd6d3c8 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c @@ -1271,7 +1271,7 @@ perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) struct event_constraint emptyconstraint; struct event_constraint unconstrained; -static int __cpuinit +static int x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (long)hcpu; diff --git a/arch/x86/kernel/cpu/perf_event_amd_ibs.c b/arch/x86/kernel/cpu/perf_event_amd_ibs.c index b46601ada8134..4b8e4d3cd6ea6 100644 --- a/arch/x86/kernel/cpu/perf_event_amd_ibs.c +++ b/arch/x86/kernel/cpu/perf_event_amd_ibs.c @@ -894,7 +894,7 @@ static inline void perf_ibs_pm_init(void) { } #endif -static int __cpuinit +static int perf_ibs_cpu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { switch (action & ~CPU_TASKS_FROZEN) { diff --git a/arch/x86/kernel/cpu/perf_event_amd_uncore.c b/arch/x86/kernel/cpu/perf_event_amd_uncore.c index c0c661adf03e2..b7b2e3de95e39 100644 --- a/arch/x86/kernel/cpu/perf_event_amd_uncore.c +++ b/arch/x86/kernel/cpu/perf_event_amd_uncore.c @@ -288,13 +288,13 @@ static struct pmu amd_l2_pmu = { .read = amd_uncore_read, }; -static struct amd_uncore * __cpuinit amd_uncore_alloc(unsigned int cpu) +static struct amd_uncore * amd_uncore_alloc(unsigned int cpu) { return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL, cpu_to_node(cpu)); } -static void __cpuinit amd_uncore_cpu_up_prepare(unsigned int cpu) +static void amd_uncore_cpu_up_prepare(unsigned int cpu) { struct amd_uncore *uncore; @@ -322,7 +322,7 @@ static void __cpuinit amd_uncore_cpu_up_prepare(unsigned int cpu) } static struct amd_uncore * -__cpuinit amd_uncore_find_online_sibling(struct amd_uncore *this, +amd_uncore_find_online_sibling(struct amd_uncore *this, struct amd_uncore * __percpu *uncores) { unsigned int cpu; @@ -348,7 +348,7 @@ __cpuinit amd_uncore_find_online_sibling(struct amd_uncore *this, return this; } -static void __cpuinit amd_uncore_cpu_starting(unsigned int cpu) +static void amd_uncore_cpu_starting(unsigned int cpu) { unsigned int eax, ebx, ecx, edx; struct amd_uncore *uncore; @@ -376,7 +376,7 @@ static void __cpuinit amd_uncore_cpu_starting(unsigned int cpu) } } -static void __cpuinit uncore_online(unsigned int cpu, +static void uncore_online(unsigned int cpu, struct amd_uncore * __percpu *uncores) { struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu); @@ -388,7 +388,7 @@ static void __cpuinit uncore_online(unsigned int cpu, cpumask_set_cpu(cpu, uncore->active_mask); } -static void __cpuinit amd_uncore_cpu_online(unsigned int cpu) +static void amd_uncore_cpu_online(unsigned int cpu) { if (amd_uncore_nb) uncore_online(cpu, amd_uncore_nb); @@ -397,7 +397,7 @@ static void __cpuinit amd_uncore_cpu_online(unsigned int cpu) uncore_online(cpu, amd_uncore_l2); } -static void __cpuinit uncore_down_prepare(unsigned int cpu, +static void uncore_down_prepare(unsigned int cpu, struct amd_uncore * __percpu *uncores) { unsigned int i; @@ -423,7 +423,7 @@ static void __cpuinit uncore_down_prepare(unsigned int cpu, } } -static void __cpuinit amd_uncore_cpu_down_prepare(unsigned int cpu) +static void amd_uncore_cpu_down_prepare(unsigned int cpu) { if (amd_uncore_nb) uncore_down_prepare(cpu, amd_uncore_nb); @@ -432,7 +432,7 @@ static void __cpuinit amd_uncore_cpu_down_prepare(unsigned int cpu) uncore_down_prepare(cpu, amd_uncore_l2); } -static void __cpuinit uncore_dead(unsigned int cpu, +static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores) { struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu); @@ -445,7 +445,7 @@ static void __cpuinit uncore_dead(unsigned int cpu, *per_cpu_ptr(amd_uncore_nb, cpu) = NULL; } -static void __cpuinit amd_uncore_cpu_dead(unsigned int cpu) +static void amd_uncore_cpu_dead(unsigned int cpu) { if (amd_uncore_nb) uncore_dead(cpu, amd_uncore_nb); @@ -454,7 +454,7 @@ static void __cpuinit amd_uncore_cpu_dead(unsigned int cpu) uncore_dead(cpu, amd_uncore_l2); } -static int __cpuinit +static int amd_uncore_cpu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { @@ -489,7 +489,7 @@ amd_uncore_cpu_notifier(struct notifier_block *self, unsigned long action, return NOTIFY_OK; } -static struct notifier_block amd_uncore_cpu_notifier_block __cpuinitdata = { +static struct notifier_block amd_uncore_cpu_notifier_block = { .notifier_call = amd_uncore_cpu_notifier, .priority = CPU_PRI_PERF + 1, }; diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.c b/arch/x86/kernel/cpu/perf_event_intel_uncore.c index 7185af255fb52..afaf184259b94 100644 --- a/arch/x86/kernel/cpu/perf_event_intel_uncore.c +++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.c @@ -3311,7 +3311,7 @@ static void __init uncore_pci_exit(void) /* CPU hot plug/unplug are serialized by cpu_add_remove_lock mutex */ static LIST_HEAD(boxes_to_free); -static void __cpuinit uncore_kfree_boxes(void) +static void uncore_kfree_boxes(void) { struct intel_uncore_box *box; @@ -3323,7 +3323,7 @@ static void __cpuinit uncore_kfree_boxes(void) } } -static void __cpuinit uncore_cpu_dying(int cpu) +static void uncore_cpu_dying(int cpu) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; @@ -3342,7 +3342,7 @@ static void __cpuinit uncore_cpu_dying(int cpu) } } -static int __cpuinit uncore_cpu_starting(int cpu) +static int uncore_cpu_starting(int cpu) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; @@ -3385,7 +3385,7 @@ static int __cpuinit uncore_cpu_starting(int cpu) return 0; } -static int __cpuinit uncore_cpu_prepare(int cpu, int phys_id) +static int uncore_cpu_prepare(int cpu, int phys_id) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; @@ -3411,7 +3411,7 @@ static int __cpuinit uncore_cpu_prepare(int cpu, int phys_id) return 0; } -static void __cpuinit +static void uncore_change_context(struct intel_uncore_type **uncores, int old_cpu, int new_cpu) { struct intel_uncore_type *type; @@ -3449,7 +3449,7 @@ uncore_change_context(struct intel_uncore_type **uncores, int old_cpu, int new_c } } -static void __cpuinit uncore_event_exit_cpu(int cpu) +static void uncore_event_exit_cpu(int cpu) { int i, phys_id, target; @@ -3477,7 +3477,7 @@ static void __cpuinit uncore_event_exit_cpu(int cpu) uncore_change_context(pci_uncores, cpu, target); } -static void __cpuinit uncore_event_init_cpu(int cpu) +static void uncore_event_init_cpu(int cpu) { int i, phys_id; @@ -3494,7 +3494,7 @@ static void __cpuinit uncore_event_init_cpu(int cpu) } static int - __cpuinit uncore_cpu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) + uncore_cpu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (long)hcpu; @@ -3534,7 +3534,7 @@ static int return NOTIFY_OK; } -static struct notifier_block uncore_cpu_nb __cpuinitdata = { +static struct notifier_block uncore_cpu_nb = { .notifier_call = uncore_cpu_notifier, /* * to migrate uncore events, our notifier should be executed diff --git a/arch/x86/kernel/cpu/rdrand.c b/arch/x86/kernel/cpu/rdrand.c index feca286c2bb4a..88db010845cb2 100644 --- a/arch/x86/kernel/cpu/rdrand.c +++ b/arch/x86/kernel/cpu/rdrand.c @@ -52,7 +52,7 @@ static inline int rdrand_long(unsigned long *v) */ #define RESEED_LOOP ((512*128)/sizeof(unsigned long)) -void __cpuinit x86_init_rdrand(struct cpuinfo_x86 *c) +void x86_init_rdrand(struct cpuinfo_x86 *c) { #ifdef CONFIG_ARCH_RANDOM unsigned long tmp; diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index d92b5dad15dd4..f2cc63e9cf08f 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -24,13 +24,13 @@ enum cpuid_regs { CR_EBX }; -void __cpuinit init_scattered_cpuid_features(struct cpuinfo_x86 *c) +void init_scattered_cpuid_features(struct cpuinfo_x86 *c) { u32 max_level; u32 regs[4]; const struct cpuid_bit *cb; - static const struct cpuid_bit __cpuinitconst cpuid_bits[] = { + static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_DTHERM, CR_EAX, 0, 0x00000006, 0 }, { X86_FEATURE_IDA, CR_EAX, 1, 0x00000006, 0 }, { X86_FEATURE_ARAT, CR_EAX, 2, 0x00000006, 0 }, diff --git a/arch/x86/kernel/cpu/topology.c b/arch/x86/kernel/cpu/topology.c index 4397e987a1cfb..4c60eaf0571c2 100644 --- a/arch/x86/kernel/cpu/topology.c +++ b/arch/x86/kernel/cpu/topology.c @@ -26,7 +26,7 @@ * exists, use it for populating initial_apicid and cpu topology * detection. */ -void __cpuinit detect_extended_topology(struct cpuinfo_x86 *c) +void detect_extended_topology(struct cpuinfo_x86 *c) { #ifdef CONFIG_SMP unsigned int eax, ebx, ecx, edx, sub_index; diff --git a/arch/x86/kernel/cpu/transmeta.c b/arch/x86/kernel/cpu/transmeta.c index 28000743bbb06..aa0430d69b900 100644 --- a/arch/x86/kernel/cpu/transmeta.c +++ b/arch/x86/kernel/cpu/transmeta.c @@ -5,7 +5,7 @@ #include #include "cpu.h" -static void __cpuinit early_init_transmeta(struct cpuinfo_x86 *c) +static void early_init_transmeta(struct cpuinfo_x86 *c) { u32 xlvl; @@ -17,7 +17,7 @@ static void __cpuinit early_init_transmeta(struct cpuinfo_x86 *c) } } -static void __cpuinit init_transmeta(struct cpuinfo_x86 *c) +static void init_transmeta(struct cpuinfo_x86 *c) { unsigned int cap_mask, uk, max, dummy; unsigned int cms_rev1, cms_rev2; @@ -98,7 +98,7 @@ static void __cpuinit init_transmeta(struct cpuinfo_x86 *c) #endif } -static const struct cpu_dev __cpuinitconst transmeta_cpu_dev = { +static const struct cpu_dev transmeta_cpu_dev = { .c_vendor = "Transmeta", .c_ident = { "GenuineTMx86", "TransmetaCPU" }, .c_early_init = early_init_transmeta, diff --git a/arch/x86/kernel/cpu/umc.c b/arch/x86/kernel/cpu/umc.c index fd2c37bf7acb6..202759a141212 100644 --- a/arch/x86/kernel/cpu/umc.c +++ b/arch/x86/kernel/cpu/umc.c @@ -8,7 +8,7 @@ * so no special init takes place. */ -static const struct cpu_dev __cpuinitconst umc_cpu_dev = { +static const struct cpu_dev umc_cpu_dev = { .c_vendor = "UMC", .c_ident = { "UMC UMC UMC" }, .c_models = { diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c index 03a36321ec544..7076878404ec0 100644 --- a/arch/x86/kernel/cpu/vmware.c +++ b/arch/x86/kernel/cpu/vmware.c @@ -122,7 +122,7 @@ static bool __init vmware_platform(void) * so that the kernel could just trust the hypervisor with providing a * reliable virtual TSC that is suitable for timekeeping. */ -static void __cpuinit vmware_set_cpu_features(struct cpuinfo_x86 *c) +static void vmware_set_cpu_features(struct cpuinfo_x86 *c) { set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC); set_cpu_cap(c, X86_FEATURE_TSC_RELIABLE); diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c index 1e4dbcfe6d316..563fea5fcba26 100644 --- a/arch/x86/kernel/cpuid.c +++ b/arch/x86/kernel/cpuid.c @@ -137,7 +137,7 @@ static const struct file_operations cpuid_fops = { .open = cpuid_open, }; -static __cpuinit int cpuid_device_create(int cpu) +static int cpuid_device_create(int cpu) { struct device *dev; @@ -151,7 +151,7 @@ static void cpuid_device_destroy(int cpu) device_destroy(cpuid_class, MKDEV(CPUID_MAJOR, cpu)); } -static int __cpuinit cpuid_class_cpu_callback(struct notifier_block *nfb, +static int cpuid_class_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { diff --git a/arch/x86/kernel/devicetree.c b/arch/x86/kernel/devicetree.c index 2fbad6b9f23c0..a4aad73cd57f3 100644 --- a/arch/x86/kernel/devicetree.c +++ b/arch/x86/kernel/devicetree.c @@ -132,7 +132,7 @@ static void x86_of_pci_irq_disable(struct pci_dev *dev) { } -void __cpuinit x86_of_pci_init(void) +void x86_of_pci_init(void) { pcibios_enable_irq = x86_of_pci_irq_enable; pcibios_disable_irq = x86_of_pci_irq_disable; diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c index b03ff1842547a..e6f08c6853f7c 100644 --- a/arch/x86/kernel/i387.c +++ b/arch/x86/kernel/i387.c @@ -119,7 +119,7 @@ unsigned int xstate_size; EXPORT_SYMBOL_GPL(xstate_size); static struct i387_fxsave_struct fx_scratch __cpuinitdata; -static void __cpuinit mxcsr_feature_mask_init(void) +static void mxcsr_feature_mask_init(void) { unsigned long mask = 0; @@ -133,7 +133,7 @@ static void __cpuinit mxcsr_feature_mask_init(void) mxcsr_feature_mask &= mask; } -static void __cpuinit init_thread_xstate(void) +static void init_thread_xstate(void) { /* * Note that xstate_size might be overwriten later during @@ -162,7 +162,7 @@ static void __cpuinit init_thread_xstate(void) * into all processes. */ -void __cpuinit fpu_init(void) +void fpu_init(void) { unsigned long cr0; unsigned long cr4_mask = 0; diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c index 344faf8d0d621..4186755f1d7cf 100644 --- a/arch/x86/kernel/irq_32.c +++ b/arch/x86/kernel/irq_32.c @@ -119,7 +119,7 @@ execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq) /* * allocate per-cpu stacks for hardirq and for softirq processing */ -void __cpuinit irq_ctx_init(int cpu) +void irq_ctx_init(int cpu) { union irq_ctx *irqctx; diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index c4ff2a9161399..d561f0a5b33a9 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -327,7 +327,7 @@ static void kvm_guest_apic_eoi_write(u32 reg, u32 val) apic_write(APIC_EOI, APIC_EOI_ACK); } -void __cpuinit kvm_guest_cpu_init(void) +void kvm_guest_cpu_init(void) { if (!kvm_para_available()) return; @@ -428,7 +428,7 @@ static void __init kvm_smp_prepare_boot_cpu(void) native_smp_prepare_boot_cpu(); } -static void __cpuinit kvm_guest_cpu_online(void *dummy) +static void kvm_guest_cpu_online(void *dummy) { kvm_guest_cpu_init(); } @@ -442,7 +442,7 @@ static void kvm_guest_cpu_offline(void *dummy) apf_task_wake_all(); } -static int __cpuinit kvm_cpu_notify(struct notifier_block *self, +static int kvm_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { int cpu = (unsigned long)hcpu; @@ -462,7 +462,7 @@ static int __cpuinit kvm_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata kvm_cpu_notifier = { +static struct notifier_block kvm_cpu_notifier = { .notifier_call = kvm_cpu_notify, }; #endif diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index 41514f56c2413..836c78b7ac1d0 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -185,7 +185,7 @@ static void kvm_restore_sched_clock_state(void) } #ifdef CONFIG_X86_LOCAL_APIC -static void __cpuinit kvm_setup_secondary_clock(void) +static void kvm_setup_secondary_clock(void) { /* * Now that the first cpu already had this clocksource initialized, diff --git a/arch/x86/kernel/microcode_core.c b/arch/x86/kernel/microcode_core.c index 22db92bbdf1a4..15c987698b0f7 100644 --- a/arch/x86/kernel/microcode_core.c +++ b/arch/x86/kernel/microcode_core.c @@ -468,7 +468,7 @@ static struct syscore_ops mc_syscore_ops = { .resume = mc_bp_resume, }; -static __cpuinit int +static int mc_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; diff --git a/arch/x86/kernel/microcode_core_early.c b/arch/x86/kernel/microcode_core_early.c index 833d51d6ee065..c97844e3aedbb 100644 --- a/arch/x86/kernel/microcode_core_early.c +++ b/arch/x86/kernel/microcode_core_early.c @@ -40,7 +40,7 @@ * * x86_vendor() gets vendor information directly through cpuid. */ -static int __cpuinit x86_vendor(void) +static int x86_vendor(void) { u32 eax = 0x00000000; u32 ebx, ecx = 0, edx; @@ -56,7 +56,7 @@ static int __cpuinit x86_vendor(void) return X86_VENDOR_UNKNOWN; } -static int __cpuinit x86_family(void) +static int x86_family(void) { u32 eax = 0x00000001; u32 ebx, ecx = 0, edx; @@ -85,7 +85,7 @@ void __init load_ucode_bsp(void) load_ucode_intel_bsp(); } -void __cpuinit load_ucode_ap(void) +void load_ucode_ap(void) { int vendor, x86; diff --git a/arch/x86/kernel/microcode_intel_early.c b/arch/x86/kernel/microcode_intel_early.c index a883942aee44a..a1f9ebf459165 100644 --- a/arch/x86/kernel/microcode_intel_early.c +++ b/arch/x86/kernel/microcode_intel_early.c @@ -34,7 +34,7 @@ struct mc_saved_data { struct microcode_intel **mc_saved; } mc_saved_data; -static enum ucode_state __cpuinit +static enum ucode_state generic_load_microcode_early(struct microcode_intel **mc_saved_p, unsigned int mc_saved_count, struct ucode_cpu_info *uci) @@ -69,7 +69,7 @@ generic_load_microcode_early(struct microcode_intel **mc_saved_p, return state; } -static void __cpuinit +static void microcode_pointer(struct microcode_intel **mc_saved, unsigned long *mc_saved_in_initrd, unsigned long initrd_start, int mc_saved_count) @@ -82,7 +82,7 @@ microcode_pointer(struct microcode_intel **mc_saved, } #ifdef CONFIG_X86_32 -static void __cpuinit +static void microcode_phys(struct microcode_intel **mc_saved_tmp, struct mc_saved_data *mc_saved_data) { @@ -101,7 +101,7 @@ microcode_phys(struct microcode_intel **mc_saved_tmp, } #endif -static enum ucode_state __cpuinit +static enum ucode_state load_microcode(struct mc_saved_data *mc_saved_data, unsigned long *mc_saved_in_initrd, unsigned long initrd_start, @@ -375,7 +375,7 @@ do { \ #define native_wrmsr(msr, low, high) \ native_write_msr(msr, low, high); -static int __cpuinit collect_cpu_info_early(struct ucode_cpu_info *uci) +static int collect_cpu_info_early(struct ucode_cpu_info *uci) { unsigned int val[2]; u8 x86, x86_model; @@ -584,7 +584,7 @@ scan_microcode(unsigned long start, unsigned long end, /* * Print ucode update info. */ -static void __cpuinit +static void print_ucode_info(struct ucode_cpu_info *uci, unsigned int date) { int cpu = smp_processor_id(); @@ -605,7 +605,7 @@ static int current_mc_date; /* * Print early updated ucode info after printk works. This is delayed info dump. */ -void __cpuinit show_ucode_info_early(void) +void show_ucode_info_early(void) { struct ucode_cpu_info uci; @@ -621,7 +621,7 @@ void __cpuinit show_ucode_info_early(void) * mc_saved_data.mc_saved and delay printing microcode info in * show_ucode_info_early() until printk() works. */ -static void __cpuinit print_ucode(struct ucode_cpu_info *uci) +static void print_ucode(struct ucode_cpu_info *uci) { struct microcode_intel *mc_intel; int *delay_ucode_info_p; @@ -643,12 +643,12 @@ static void __cpuinit print_ucode(struct ucode_cpu_info *uci) * Flush global tlb. We only do this in x86_64 where paging has been enabled * already and PGE should be enabled as well. */ -static inline void __cpuinit flush_tlb_early(void) +static inline void flush_tlb_early(void) { __native_flush_tlb_global_irq_disabled(); } -static inline void __cpuinit print_ucode(struct ucode_cpu_info *uci) +static inline void print_ucode(struct ucode_cpu_info *uci) { struct microcode_intel *mc_intel; @@ -660,7 +660,7 @@ static inline void __cpuinit print_ucode(struct ucode_cpu_info *uci) } #endif -static int __cpuinit apply_microcode_early(struct mc_saved_data *mc_saved_data, +static int apply_microcode_early(struct mc_saved_data *mc_saved_data, struct ucode_cpu_info *uci) { struct microcode_intel *mc_intel; @@ -763,7 +763,7 @@ load_ucode_intel_bsp(void) #endif } -void __cpuinit load_ucode_intel_ap(void) +void load_ucode_intel_ap(void) { struct mc_saved_data *mc_saved_data_p; struct ucode_cpu_info uci; diff --git a/arch/x86/kernel/mmconf-fam10h_64.c b/arch/x86/kernel/mmconf-fam10h_64.c index ac861b8348e2d..33fd4e6407b7b 100644 --- a/arch/x86/kernel/mmconf-fam10h_64.c +++ b/arch/x86/kernel/mmconf-fam10h_64.c @@ -24,14 +24,14 @@ struct pci_hostbridge_probe { u32 device; }; -static u64 __cpuinitdata fam10h_pci_mmconf_base; +static u64 fam10h_pci_mmconf_base; -static struct pci_hostbridge_probe pci_probes[] __cpuinitdata = { +static struct pci_hostbridge_probe pci_probes[] = { { 0, 0x18, PCI_VENDOR_ID_AMD, 0x1200 }, { 0xff, 0, PCI_VENDOR_ID_AMD, 0x1200 }, }; -static int __cpuinit cmp_range(const void *x1, const void *x2) +static int cmp_range(const void *x1, const void *x2) { const struct range *r1 = x1; const struct range *r2 = x2; @@ -49,7 +49,7 @@ static int __cpuinit cmp_range(const void *x1, const void *x2) /* need to avoid (0xfd<<32), (0xfe<<32), and (0xff<<32), ht used space */ #define FAM10H_PCI_MMCONF_BASE (0xfcULL<<32) #define BASE_VALID(b) ((b) + MMCONF_SIZE <= (0xfdULL<<32) || (b) >= (1ULL<<40)) -static void __cpuinit get_fam10h_pci_mmconf_base(void) +static void get_fam10h_pci_mmconf_base(void) { int i; unsigned bus; @@ -166,7 +166,7 @@ static void __cpuinit get_fam10h_pci_mmconf_base(void) fam10h_pci_mmconf_base = base; } -void __cpuinit fam10h_check_enable_mmcfg(void) +void fam10h_check_enable_mmcfg(void) { u64 val; u32 address; @@ -230,7 +230,7 @@ static const struct dmi_system_id __initconst mmconf_dmi_table[] = { {} }; -/* Called from a __cpuinit function, but only on the BSP. */ +/* Called from a function, but only on the BSP. */ void __ref check_enable_amd_mmconf_dmi(void) { dmi_check_system(mmconf_dmi_table); diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c index ce130493b8020..6d9d08e443c57 100644 --- a/arch/x86/kernel/msr.c +++ b/arch/x86/kernel/msr.c @@ -200,7 +200,7 @@ static const struct file_operations msr_fops = { .compat_ioctl = msr_ioctl, }; -static int __cpuinit msr_device_create(int cpu) +static int msr_device_create(int cpu) { struct device *dev; @@ -214,7 +214,7 @@ static void msr_device_destroy(int cpu) device_destroy(msr_class, MKDEV(MSR_MAJOR, cpu)); } -static int __cpuinit msr_class_cpu_callback(struct notifier_block *nfb, +static int msr_class_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 48f4399534361..3179fc1bfd970 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -385,7 +385,7 @@ static void amd_e400_idle(void) default_idle(); } -void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c) +void select_idle_routine(const struct cpuinfo_x86 *c) { #ifdef CONFIG_SMP if (boot_option_idle_override == IDLE_POLL && smp_num_siblings > 1) diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index e4baa389ee502..8aac112426baa 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -170,7 +170,7 @@ static struct resource bss_resource = { #ifdef CONFIG_X86_32 /* cpu data as detected by the assembly code in head.S */ -struct cpuinfo_x86 new_cpu_data __cpuinitdata = { +struct cpuinfo_x86 new_cpu_data = { .wp_works_ok = -1, }; /* common cpu data for all cpus */ diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 87084ab90d190..009e1adaf9073 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -130,7 +130,7 @@ atomic_t init_deasserted; * Report back to the Boot Processor during boot time or to the caller processor * during CPU online. */ -static void __cpuinit smp_callin(void) +static void smp_callin(void) { int cpuid, phys_id; unsigned long timeout; @@ -237,7 +237,7 @@ static int enable_start_cpu0; /* * Activate a secondary processor. */ -notrace static void __cpuinit start_secondary(void *unused) +notrace static void start_secondary(void *unused) { /* * Don't put *anything* before cpu_init(), SMP booting is too @@ -307,7 +307,7 @@ void __init smp_store_boot_cpu_info(void) * The bootstrap kernel entry code has set these up. Save them for * a given CPU */ -void __cpuinit smp_store_cpu_info(int id) +void smp_store_cpu_info(int id) { struct cpuinfo_x86 *c = &cpu_data(id); @@ -320,7 +320,7 @@ void __cpuinit smp_store_cpu_info(int id) identify_secondary_cpu(c); } -static bool __cpuinit +static bool topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name) { int cpu1 = c->cpu_index, cpu2 = o->cpu_index; @@ -337,7 +337,7 @@ do { \ cpumask_set_cpu((c2), cpu_##_m##_mask(c1)); \ } while (0) -static bool __cpuinit match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) +static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) { if (cpu_has_topoext) { int cpu1 = c->cpu_index, cpu2 = o->cpu_index; @@ -355,7 +355,7 @@ static bool __cpuinit match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) return false; } -static bool __cpuinit match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) +static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) { int cpu1 = c->cpu_index, cpu2 = o->cpu_index; @@ -366,7 +366,7 @@ static bool __cpuinit match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) return false; } -static bool __cpuinit match_mc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) +static bool match_mc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) { if (c->phys_proc_id == o->phys_proc_id) { if (cpu_has(c, X86_FEATURE_AMD_DCM)) @@ -377,7 +377,7 @@ static bool __cpuinit match_mc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) return false; } -void __cpuinit set_cpu_sibling_map(int cpu) +void set_cpu_sibling_map(int cpu) { bool has_smt = smp_num_siblings > 1; bool has_mp = has_smt || boot_cpu_data.x86_max_cores > 1; @@ -506,7 +506,7 @@ void __inquire_remote_apic(int apicid) * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this * won't ... remember to clear down the APIC, etc later. */ -int __cpuinit +int wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip) { unsigned long send_status, accept_status = 0; @@ -540,7 +540,7 @@ wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip) return (send_status | accept_status); } -static int __cpuinit +static int wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip) { unsigned long send_status, accept_status = 0; @@ -656,7 +656,7 @@ wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip) } /* reduce the number of lines printed when booting a large cpu count system */ -static void __cpuinit announce_cpu(int cpu, int apicid) +static void announce_cpu(int cpu, int apicid) { static int current_node = -1; int node = early_cpu_to_node(cpu); @@ -698,7 +698,7 @@ static int wakeup_cpu0_nmi(unsigned int cmd, struct pt_regs *regs) * We'll change this code in the future to wake up hard offlined CPU0 if * real platform and request are available. */ -static int __cpuinit +static int wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid, int *cpu0_nmi_registered) { @@ -738,7 +738,7 @@ wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid, * Returns zero if CPU booted OK, else error code from * ->wakeup_secondary_cpu. */ -static int __cpuinit do_boot_cpu(int apicid, int cpu, struct task_struct *idle) +static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle) { volatile u32 *trampoline_status = (volatile u32 *) __va(real_mode_header->trampoline_status); @@ -879,7 +879,7 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu, struct task_struct *idle) return boot_error; } -int __cpuinit native_cpu_up(unsigned int cpu, struct task_struct *tidle) +int native_cpu_up(unsigned int cpu, struct task_struct *tidle) { int apicid = apic->cpu_present_to_apicid(cpu); unsigned long flags; diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c index f84fe00fad48a..68e8057f03c6b 100644 --- a/arch/x86/kernel/tboot.c +++ b/arch/x86/kernel/tboot.c @@ -319,7 +319,7 @@ static int tboot_wait_for_aps(int num_aps) return !(atomic_read((atomic_t *)&tboot->num_in_wfs) == num_aps); } -static int __cpuinit tboot_cpu_callback(struct notifier_block *nfb, +static int tboot_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { switch (action) { @@ -333,7 +333,7 @@ static int __cpuinit tboot_cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block tboot_cpu_notifier __cpuinitdata = +static struct notifier_block tboot_cpu_notifier = { .notifier_call = tboot_cpu_callback, }; diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 9714a7aa32fcd..3dc7633636593 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -827,7 +827,7 @@ static void __init check_system_tsc_reliable(void) * Make an educated guess if the TSC is trustworthy and synchronized * over all CPUs. */ -__cpuinit int unsynchronized_tsc(void) +int unsynchronized_tsc(void) { if (!cpu_has_tsc || tsc_unstable) return 1; @@ -1026,7 +1026,7 @@ void __init tsc_init(void) * been calibrated. This assumes that CONSTANT_TSC applies to all * cpus in the socket - this should be a safe assumption. */ -unsigned long __cpuinit calibrate_delay_is_known(void) +unsigned long calibrate_delay_is_known(void) { int i, cpu = smp_processor_id(); diff --git a/arch/x86/kernel/tsc_sync.c b/arch/x86/kernel/tsc_sync.c index fc25e60a58845..adfdf56a3714e 100644 --- a/arch/x86/kernel/tsc_sync.c +++ b/arch/x86/kernel/tsc_sync.c @@ -25,24 +25,24 @@ * Entry/exit counters that make sure that both CPUs * run the measurement code at once: */ -static __cpuinitdata atomic_t start_count; -static __cpuinitdata atomic_t stop_count; +static atomic_t start_count; +static atomic_t stop_count; /* * We use a raw spinlock in this exceptional case, because * we want to have the fastest, inlined, non-debug version * of a critical section, to be able to prove TSC time-warps: */ -static __cpuinitdata arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED; +static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED; -static __cpuinitdata cycles_t last_tsc; -static __cpuinitdata cycles_t max_warp; -static __cpuinitdata int nr_warps; +static cycles_t last_tsc; +static cycles_t max_warp; +static int nr_warps; /* * TSC-warp measurement loop running on both CPUs: */ -static __cpuinit void check_tsc_warp(unsigned int timeout) +static void check_tsc_warp(unsigned int timeout) { cycles_t start, now, prev, end; int i; @@ -121,7 +121,7 @@ static inline unsigned int loop_timeout(int cpu) * Source CPU calls into this - it waits for the freshly booted * target CPU to arrive and then starts the measurement: */ -void __cpuinit check_tsc_sync_source(int cpu) +void check_tsc_sync_source(int cpu) { int cpus = 2; @@ -187,7 +187,7 @@ void __cpuinit check_tsc_sync_source(int cpu) /* * Freshly booted CPUs call into this: */ -void __cpuinit check_tsc_sync_target(void) +void check_tsc_sync_target(void) { int cpus = 2; diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c index c52c07efe970a..09ce23ae370cc 100644 --- a/arch/x86/kernel/vsyscall_64.c +++ b/arch/x86/kernel/vsyscall_64.c @@ -331,7 +331,7 @@ bool emulate_vsyscall(struct pt_regs *regs, unsigned long address) * Assume __initcall executes before all user space. Hopefully kmod * doesn't violate that. We'll find out if it does. */ -static void __cpuinit vsyscall_set_cpu(int cpu) +static void vsyscall_set_cpu(int cpu) { unsigned long d; unsigned long node = 0; @@ -353,13 +353,13 @@ static void __cpuinit vsyscall_set_cpu(int cpu) write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S); } -static void __cpuinit cpu_vsyscall_init(void *arg) +static void cpu_vsyscall_init(void *arg) { /* preemption should be already off */ vsyscall_set_cpu(raw_smp_processor_id()); } -static int __cpuinit +static int cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg) { long cpu = (long)arg; diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c index 45a14dbbddafe..5f24c71accaaf 100644 --- a/arch/x86/kernel/x86_init.c +++ b/arch/x86/kernel/x86_init.c @@ -25,7 +25,7 @@ #include #include -void __cpuinit x86_init_noop(void) { } +void x86_init_noop(void) { } void __init x86_init_uint_noop(unsigned int unused) { } int __init iommu_init_noop(void) { return 0; } void iommu_shutdown_noop(void) { } @@ -85,7 +85,7 @@ struct x86_init_ops x86_init __initdata = { }, }; -struct x86_cpuinit_ops x86_cpuinit __cpuinitdata = { +struct x86_cpuinit_ops x86_cpuinit = { .early_percpu_clock_init = x86_init_noop, .setup_percpu_clockev = setup_secondary_APIC_clock, }; diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c index 92f37e7683c50..7ed034752e934 100644 --- a/arch/x86/kernel/xsave.c +++ b/arch/x86/kernel/xsave.c @@ -576,7 +576,7 @@ static void __init xstate_enable_boot_cpu(void) * This is somewhat obfuscated due to the lack of powerful enough * overrides for the section checks. */ -void __cpuinit xsave_init(void) +void xsave_init(void) { static __refdata void (*next_func)(void) = xstate_enable_boot_cpu; void (*this_func)(void); @@ -597,7 +597,7 @@ static inline void __init eager_fpu_init_bp(void) setup_init_fpu_buf(); } -void __cpuinit eager_fpu_init(void) +void eager_fpu_init(void) { static __refdata void (*boot_func)(void) = eager_fpu_init_bp; diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c index a71c4e207679a..8bf93bae1f13f 100644 --- a/arch/x86/mm/numa.c +++ b/arch/x86/mm/numa.c @@ -60,7 +60,7 @@ s16 __apicid_to_node[MAX_LOCAL_APIC] = { [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE }; -int __cpuinit numa_cpu_node(int cpu) +int numa_cpu_node(int cpu) { int apicid = early_per_cpu(x86_cpu_to_apicid, cpu); @@ -691,12 +691,12 @@ void __init init_cpu_to_node(void) #ifndef CONFIG_DEBUG_PER_CPU_MAPS # ifndef CONFIG_NUMA_EMU -void __cpuinit numa_add_cpu(int cpu) +void numa_add_cpu(int cpu) { cpumask_set_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]); } -void __cpuinit numa_remove_cpu(int cpu) +void numa_remove_cpu(int cpu) { cpumask_clear_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]); } @@ -763,17 +763,17 @@ void debug_cpumask_set_cpu(int cpu, int node, bool enable) } # ifndef CONFIG_NUMA_EMU -static void __cpuinit numa_set_cpumask(int cpu, bool enable) +static void numa_set_cpumask(int cpu, bool enable) { debug_cpumask_set_cpu(cpu, early_cpu_to_node(cpu), enable); } -void __cpuinit numa_add_cpu(int cpu) +void numa_add_cpu(int cpu) { numa_set_cpumask(cpu, true); } -void __cpuinit numa_remove_cpu(int cpu) +void numa_remove_cpu(int cpu) { numa_set_cpumask(cpu, false); } diff --git a/arch/x86/mm/numa_emulation.c b/arch/x86/mm/numa_emulation.c index dbbbb47260ccc..67c2d21b800b9 100644 --- a/arch/x86/mm/numa_emulation.c +++ b/arch/x86/mm/numa_emulation.c @@ -444,7 +444,7 @@ void __init numa_emulation(struct numa_meminfo *numa_meminfo, int numa_dist_cnt) } #ifndef CONFIG_DEBUG_PER_CPU_MAPS -void __cpuinit numa_add_cpu(int cpu) +void numa_add_cpu(int cpu) { int physnid, nid; @@ -462,7 +462,7 @@ void __cpuinit numa_add_cpu(int cpu) cpumask_set_cpu(cpu, node_to_cpumask_map[nid]); } -void __cpuinit numa_remove_cpu(int cpu) +void numa_remove_cpu(int cpu) { int i; @@ -470,7 +470,7 @@ void __cpuinit numa_remove_cpu(int cpu) cpumask_clear_cpu(cpu, node_to_cpumask_map[i]); } #else /* !CONFIG_DEBUG_PER_CPU_MAPS */ -static void __cpuinit numa_set_cpumask(int cpu, bool enable) +static void numa_set_cpumask(int cpu, bool enable) { int nid, physnid; @@ -490,12 +490,12 @@ static void __cpuinit numa_set_cpumask(int cpu, bool enable) } } -void __cpuinit numa_add_cpu(int cpu) +void numa_add_cpu(int cpu) { numa_set_cpumask(cpu, true); } -void __cpuinit numa_remove_cpu(int cpu) +void numa_remove_cpu(int cpu) { numa_set_cpumask(cpu, false); } diff --git a/arch/x86/mm/setup_nx.c b/arch/x86/mm/setup_nx.c index 410531d3c292d..da23af2868821 100644 --- a/arch/x86/mm/setup_nx.c +++ b/arch/x86/mm/setup_nx.c @@ -29,7 +29,7 @@ static int __init noexec_setup(char *str) } early_param("noexec", noexec_setup); -void __cpuinit x86_configure_nx(void) +void x86_configure_nx(void) { if (cpu_has_nx && !disable_nx) __supported_pte_mask |= _PAGE_NX; diff --git a/arch/x86/pci/amd_bus.c b/arch/x86/pci/amd_bus.c index e9e6ed5cdf946..a0077b78ae86c 100644 --- a/arch/x86/pci/amd_bus.c +++ b/arch/x86/pci/amd_bus.c @@ -312,7 +312,7 @@ static int __init early_fill_mp_bus_info(void) #define ENABLE_CF8_EXT_CFG (1ULL << 46) -static void __cpuinit enable_pci_io_ecs(void *unused) +static void enable_pci_io_ecs(void *unused) { u64 reg; rdmsrl(MSR_AMD64_NB_CFG, reg); @@ -322,7 +322,7 @@ static void __cpuinit enable_pci_io_ecs(void *unused) } } -static int __cpuinit amd_cpu_notify(struct notifier_block *self, +static int amd_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { int cpu = (long)hcpu; @@ -337,7 +337,7 @@ static int __cpuinit amd_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata amd_cpu_notifier = { +static struct notifier_block amd_cpu_notifier = { .notifier_call = amd_cpu_notify, }; diff --git a/arch/x86/platform/ce4100/ce4100.c b/arch/x86/platform/ce4100/ce4100.c index f8ab4945892ee..baec704231b31 100644 --- a/arch/x86/platform/ce4100/ce4100.c +++ b/arch/x86/platform/ce4100/ce4100.c @@ -134,7 +134,7 @@ static void __init sdv_arch_setup(void) } #ifdef CONFIG_X86_IO_APIC -static void __cpuinit sdv_pci_init(void) +static void sdv_pci_init(void) { x86_of_pci_init(); /* We can't set this earlier, because we need to calibrate the timer */ diff --git a/arch/x86/platform/mrst/mrst.c b/arch/x86/platform/mrst/mrst.c index a0a0a4389bbd9..47fe66fe61f12 100644 --- a/arch/x86/platform/mrst/mrst.c +++ b/arch/x86/platform/mrst/mrst.c @@ -65,7 +65,7 @@ * lapic (always-on,ARAT) ------ 150 */ -__cpuinitdata enum mrst_timer_options mrst_timer_options; +enum mrst_timer_options mrst_timer_options; static u32 sfi_mtimer_usage[SFI_MTMR_MAX_NUM]; static struct sfi_timer_table_entry sfi_mtimer_array[SFI_MTMR_MAX_NUM]; @@ -248,7 +248,7 @@ static void __init mrst_time_init(void) apbt_time_init(); } -static void __cpuinit mrst_arch_setup(void) +static void mrst_arch_setup(void) { if (boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model == 0x27) __mrst_cpu_chip = MRST_CPU_CHIP_PENWELL; diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 34511cf6baad6..a1531aa0d845f 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -1725,7 +1725,7 @@ static void __init init_hvm_pv_info(void) xen_domain_type = XEN_HVM_DOMAIN; } -static int __cpuinit xen_hvm_cpu_notify(struct notifier_block *self, +static int xen_hvm_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { int cpu = (long)hcpu; @@ -1744,7 +1744,7 @@ static int __cpuinit xen_hvm_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block xen_hvm_cpu_notifier __cpuinitdata = { +static struct notifier_block xen_hvm_cpu_notifier = { .notifier_call = xen_hvm_cpu_notify, }; diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c index 0a9fb7a0b4521..8f3eea6b80c52 100644 --- a/arch/x86/xen/setup.c +++ b/arch/x86/xen/setup.c @@ -497,7 +497,7 @@ static void __init fiddle_vdso(void) #endif } -static int __cpuinit register_callback(unsigned type, const void *func) +static int register_callback(unsigned type, const void *func) { struct callback_register callback = { .type = type, @@ -508,7 +508,7 @@ static int __cpuinit register_callback(unsigned type, const void *func) return HYPERVISOR_callback_op(CALLBACKOP_register, &callback); } -void __cpuinit xen_enable_sysenter(void) +void xen_enable_sysenter(void) { int ret; unsigned sysenter_feature; @@ -527,7 +527,7 @@ void __cpuinit xen_enable_sysenter(void) setup_clear_cpu_cap(sysenter_feature); } -void __cpuinit xen_enable_syscall(void) +void xen_enable_syscall(void) { #ifdef CONFIG_X86_64 int ret; diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c index 570c9a5c4d3f6..bee9e9b9cb4c0 100644 --- a/arch/x86/xen/smp.c +++ b/arch/x86/xen/smp.c @@ -61,7 +61,7 @@ static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static void __cpuinit cpu_bringup(void) +static void cpu_bringup(void) { int cpu; @@ -93,7 +93,7 @@ static void __cpuinit cpu_bringup(void) wmb(); /* make sure everything is out */ } -static void __cpuinit cpu_bringup_and_idle(void) +static void cpu_bringup_and_idle(void) { cpu_bringup(); cpu_startup_entry(CPUHP_ONLINE); @@ -303,7 +303,7 @@ static void __init xen_smp_prepare_cpus(unsigned int max_cpus) set_cpu_present(cpu, true); } -static int __cpuinit +static int cpu_initialize_context(unsigned int cpu, struct task_struct *idle) { struct vcpu_guest_context *ctxt; @@ -374,7 +374,7 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle) return 0; } -static int __cpuinit xen_cpu_up(unsigned int cpu, struct task_struct *idle) +static int xen_cpu_up(unsigned int cpu, struct task_struct *idle) { int rc; @@ -452,7 +452,7 @@ static void xen_cpu_die(unsigned int cpu) xen_teardown_timer(cpu); } -static void __cpuinit xen_play_dead(void) /* used only with HOTPLUG_CPU */ +static void xen_play_dead(void) /* used only with HOTPLUG_CPU */ { play_dead_common(); HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL); @@ -673,7 +673,7 @@ static void __init xen_hvm_smp_prepare_cpus(unsigned int max_cpus) xen_init_lock_cpu(0); } -static int __cpuinit xen_hvm_cpu_up(unsigned int cpu, struct task_struct *tidle) +static int xen_hvm_cpu_up(unsigned int cpu, struct task_struct *tidle) { int rc; /* diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c index 3002ec1bb71a2..9553b2b917363 100644 --- a/arch/x86/xen/spinlock.c +++ b/arch/x86/xen/spinlock.c @@ -359,7 +359,7 @@ static irqreturn_t dummy_handler(int irq, void *dev_id) return IRQ_HANDLED; } -void __cpuinit xen_init_lock_cpu(int cpu) +void xen_init_lock_cpu(int cpu) { int irq; const char *name; diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index a95b41744ad0b..86782c5d7e2a6 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h @@ -73,7 +73,7 @@ static inline void xen_hvm_smp_init(void) {} #ifdef CONFIG_PARAVIRT_SPINLOCKS void __init xen_init_spinlocks(void); -void __cpuinit xen_init_lock_cpu(int cpu); +void xen_init_lock_cpu(int cpu); void xen_uninit_lock_cpu(int cpu); #else static inline void xen_init_spinlocks(void) diff --git a/arch/xtensa/kernel/time.c b/arch/xtensa/kernel/time.c index ffb4741043116..f1818db1d9882 100644 --- a/arch/xtensa/kernel/time.c +++ b/arch/xtensa/kernel/time.c @@ -109,7 +109,7 @@ irqreturn_t timer_interrupt (int irq, void *dev_id) } #ifndef CONFIG_GENERIC_CALIBRATE_DELAY -void __cpuinit calibrate_delay(void) +void calibrate_delay(void) { loops_per_jiffy = CCOUNT_PER_JIFFY; printk("Calibrating delay loop (skipped)... " diff --git a/block/blk-iopoll.c b/block/blk-iopoll.c index 2d4d8d5674412..5b9432264e69a 100644 --- a/block/blk-iopoll.c +++ b/block/blk-iopoll.c @@ -189,7 +189,7 @@ void blk_iopoll_init(struct blk_iopoll *iop, int weight, blk_iopoll_fn *poll_fn) } EXPORT_SYMBOL(blk_iopoll_init); -static int __cpuinit blk_iopoll_cpu_notify(struct notifier_block *self, +static int blk_iopoll_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { /* @@ -209,7 +209,7 @@ static int __cpuinit blk_iopoll_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata blk_iopoll_cpu_notifier = { +static struct notifier_block blk_iopoll_cpu_notifier = { .notifier_call = blk_iopoll_cpu_notify, }; diff --git a/block/blk-softirq.c b/block/blk-softirq.c index 2b01c0b739848..76b9d2f7cc133 100644 --- a/block/blk-softirq.c +++ b/block/blk-softirq.c @@ -78,7 +78,7 @@ static int raise_blk_irq(int cpu, struct request *rq) } #endif -static int __cpuinit blk_cpu_notify(struct notifier_block *self, +static int blk_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { /* @@ -98,7 +98,7 @@ static int __cpuinit blk_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata blk_cpu_notifier = { +static struct notifier_block blk_cpu_notifier = { .notifier_call = blk_cpu_notify, }; diff --git a/drivers/acpi/processor_core.c b/drivers/acpi/processor_core.c index 164d49569aebf..a5e9f4a5b2816 100644 --- a/drivers/acpi/processor_core.c +++ b/drivers/acpi/processor_core.c @@ -253,7 +253,7 @@ static bool __init processor_physically_present(acpi_handle handle) return true; } -static void __cpuinit acpi_set_pdc_bits(u32 *buf) +static void acpi_set_pdc_bits(u32 *buf) { buf[0] = ACPI_PDC_REVISION_ID; buf[1] = 1; @@ -265,7 +265,7 @@ static void __cpuinit acpi_set_pdc_bits(u32 *buf) arch_acpi_set_pdc_bits(buf); } -static struct acpi_object_list *__cpuinit acpi_processor_alloc_pdc(void) +static struct acpi_object_list *acpi_processor_alloc_pdc(void) { struct acpi_object_list *obj_list; union acpi_object *obj; @@ -308,7 +308,7 @@ static struct acpi_object_list *__cpuinit acpi_processor_alloc_pdc(void) * _PDC is required for a BIOS-OS handshake for most of the newer * ACPI processor features. */ -static int __cpuinit +static int acpi_processor_eval_pdc(acpi_handle handle, struct acpi_object_list *pdc_in) { acpi_status status = AE_OK; @@ -336,7 +336,7 @@ acpi_processor_eval_pdc(acpi_handle handle, struct acpi_object_list *pdc_in) return status; } -void __cpuinit acpi_processor_set_pdc(acpi_handle handle) +void acpi_processor_set_pdc(acpi_handle handle) { struct acpi_object_list *obj_list; diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c index c266cdc117840..3efb90bb0f90c 100644 --- a/drivers/acpi/processor_driver.c +++ b/drivers/acpi/processor_driver.c @@ -461,7 +461,7 @@ static struct notifier_block acpi_cpu_notifier = * acpi_cpu_soft_notify(). Getting it __cpuinit{data} is difficult, the * root cause seem to be that acpi_processor_uninstall_hotplug_notify() * is in the module_exit (__exit) func. Allowing acpi_processor_start() - * to not be in __cpuinit section, but being called from __cpuinit funcs + * to not be in section, but being called from funcs * via __ref looks like the right thing to do here. */ static __ref int acpi_processor_start(struct acpi_processor *pr) @@ -524,7 +524,7 @@ static __ref int acpi_processor_start(struct acpi_processor *pr) * (cpu_data(cpu)) values, like CPU feature flags, family, model, etc. * Such things have to be put in and set up above in acpi_processor_start() */ -static int __cpuinit acpi_processor_add(struct acpi_device *device) +static int acpi_processor_add(struct acpi_device *device) { struct acpi_processor *pr = NULL; int result = 0; diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index c991fe680e589..1b8ade5ad3801 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -98,7 +98,7 @@ static int set_max_cstate(const struct dmi_system_id *id) /* Actually this shouldn't be __cpuinitdata, would be better to fix the callers to only run once -AK */ -static struct dmi_system_id __cpuinitdata processor_power_dmi_table[] = { +static struct dmi_system_id processor_power_dmi_table[] = { { set_max_cstate, "Clevo 5600D", { DMI_MATCH(DMI_BIOS_VENDOR,"Phoenix Technologies LTD"), DMI_MATCH(DMI_BIOS_VERSION,"SHE845M0.86C.0013.D.0302131307")}, @@ -1139,7 +1139,7 @@ int acpi_processor_cst_has_changed(struct acpi_processor *pr) static int acpi_processor_registered; -int __cpuinit acpi_processor_power_init(struct acpi_processor *pr) +int acpi_processor_power_init(struct acpi_processor *pr) { acpi_status status = 0; int retval; diff --git a/drivers/android/binder.c b/drivers/android/binder.c index cd7dc1de32591..7796abcfb6ed0 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -37,6 +37,7 @@ #include #include #include +#include #ifdef CONFIG_ANDROID_BINDER_IPC_32BIT #define BINDER_IPC_32BIT 1 @@ -111,8 +112,7 @@ enum { BINDER_DEBUG_PRIORITY_CAP = 1U << 14, BINDER_DEBUG_BUFFER_ALLOC_ASYNC = 1U << 15, }; -static uint32_t binder_debug_mask; - +static uint32_t binder_debug_mask = 0; module_param_named(debug_mask, binder_debug_mask, uint, S_IWUSR | S_IRUGO); static bool binder_debug_no_lock; @@ -381,6 +381,7 @@ static int task_get_unused_fd_flags(struct binder_proc *proc, int flags) struct files_struct *files = proc->files; unsigned long rlim_cur; unsigned long irqs; + int ret; if (files == NULL) return -ESRCH; @@ -391,7 +392,11 @@ static int task_get_unused_fd_flags(struct binder_proc *proc, int flags) rlim_cur = task_rlimit(proc->tsk, RLIMIT_NOFILE); unlock_task_sighand(proc->tsk, &irqs); - return __alloc_fd(files, 0, rlim_cur, flags); + preempt_enable_no_resched(); + ret = __alloc_fd(files, 0, rlim_cur, flags); + preempt_disable(); + + return ret; } /* @@ -400,8 +405,11 @@ static int task_get_unused_fd_flags(struct binder_proc *proc, int flags) static void task_fd_install( struct binder_proc *proc, unsigned int fd, struct file *file) { - if (proc->files) + if (proc->files) { + preempt_enable_no_resched(); __fd_install(proc->files, fd, file); + preempt_disable(); + } } /* @@ -429,6 +437,7 @@ static inline void binder_lock(const char *tag) { trace_binder_lock(tag); mutex_lock(&binder_main_lock); + preempt_disable(); trace_binder_locked(tag); } @@ -436,8 +445,62 @@ static inline void binder_unlock(const char *tag) { trace_binder_unlock(tag); mutex_unlock(&binder_main_lock); + preempt_enable(); } +static inline void *kzalloc_preempt_disabled(size_t size) +{ + void *ptr; + + ptr = kzalloc(size, GFP_NOWAIT); + if (ptr) + return ptr; + + preempt_enable_no_resched(); + ptr = kzalloc(size, GFP_KERNEL); + preempt_disable(); + + return ptr; +} + +static inline long copy_to_user_preempt_disabled(void __user *to, const void *from, long n) +{ + long ret; + + preempt_enable_no_resched(); + ret = copy_to_user(to, from, n); + preempt_disable(); + return ret; +} + +static inline long copy_from_user_preempt_disabled(void *to, const void __user *from, long n) +{ + long ret; + + preempt_enable_no_resched(); + ret = copy_from_user(to, from, n); + preempt_disable(); + return ret; +} + +#define get_user_preempt_disabled(x, ptr) \ +({ \ + int __ret; \ + preempt_enable_no_resched(); \ + __ret = get_user(x, ptr); \ + preempt_disable(); \ + __ret; \ +}) + +#define put_user_preempt_disabled(x, ptr) \ +({ \ + int __ret; \ + preempt_enable_no_resched(); \ + __ret = put_user(x, ptr); \ + preempt_disable(); \ + __ret; \ +}) + static void binder_set_nice(long nice) { long min_nice; @@ -571,6 +634,8 @@ static int binder_update_page_range(struct binder_proc *proc, int allocate, else mm = get_task_mm(proc->tsk); + preempt_enable_no_resched(); + if (mm) { down_write(&mm->mmap_sem); vma = proc->vma; @@ -626,6 +691,9 @@ static int binder_update_page_range(struct binder_proc *proc, int allocate, up_write(&mm->mmap_sem); mmput(mm); } + + preempt_disable(); + return 0; free_range: @@ -648,6 +716,9 @@ static int binder_update_page_range(struct binder_proc *proc, int allocate, up_write(&mm->mmap_sem); mmput(mm); } + + preempt_disable(); + return -ENOMEM; } @@ -907,7 +978,7 @@ static struct binder_node *binder_new_node(struct binder_proc *proc, return NULL; } - node = kzalloc(sizeof(*node), GFP_KERNEL); + node = kzalloc_preempt_disabled(sizeof(*node)); if (node == NULL) return NULL; binder_stats_created(BINDER_STAT_NODE); @@ -1048,7 +1119,7 @@ static struct binder_ref *binder_get_ref_for_node(struct binder_proc *proc, else return ref; } - new_ref = kzalloc(sizeof(*ref), GFP_KERNEL); + new_ref = kzalloc_preempt_disabled(sizeof(*ref)); if (new_ref == NULL) return NULL; binder_stats_created(BINDER_STAT_REF); @@ -1414,6 +1485,10 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_DEAD_REPLY; goto err_dead_binder; } + if (security_binder_transaction(proc->tsk, target_proc->tsk) < 0) { + return_error = BR_FAILED_REPLY; + goto err_invalid_target_handle; + } if (!(tr->flags & TF_ONE_WAY) && thread->transaction_stack) { struct binder_transaction *tmp; @@ -1445,14 +1520,14 @@ static void binder_transaction(struct binder_proc *proc, e->to_proc = target_proc->pid; /* TODO: reuse incoming transaction for reply */ - t = kzalloc(sizeof(*t), GFP_KERNEL); + t = kzalloc_preempt_disabled(sizeof(*t)); if (t == NULL) { return_error = BR_FAILED_REPLY; goto err_alloc_t_failed; } binder_stats_created(BINDER_STAT_TRANSACTION); - tcomplete = kzalloc(sizeof(*tcomplete), GFP_KERNEL); + tcomplete = kzalloc_preempt_disabled(sizeof(*tcomplete)); if (tcomplete == NULL) { return_error = BR_FAILED_REPLY; goto err_alloc_tcomplete_failed; @@ -1509,14 +1584,14 @@ static void binder_transaction(struct binder_proc *proc, offp = (binder_size_t *)(t->buffer->data + ALIGN(tr->data_size, sizeof(void *))); - if (copy_from_user(t->buffer->data, (const void __user *)(uintptr_t) + if (copy_from_user_preempt_disabled(t->buffer->data, (const void __user *)(uintptr_t) tr->data.ptr.buffer, tr->data_size)) { binder_user_error("%d:%d got transaction with invalid data ptr\n", proc->pid, thread->pid); return_error = BR_FAILED_REPLY; goto err_copy_data_failed; } - if (copy_from_user(offp, (const void __user *)(uintptr_t) + if (copy_from_user_preempt_disabled(offp, (const void __user *)(uintptr_t) tr->data.ptr.offsets, tr->offsets_size)) { binder_user_error("%d:%d got transaction with invalid offsets ptr\n", proc->pid, thread->pid); @@ -1571,6 +1646,10 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_binder_get_ref_for_node_failed; } + if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) { + return_error = BR_FAILED_REPLY; + goto err_binder_get_ref_for_node_failed; + } ref = binder_get_ref_for_node(target_proc, node); if (ref == NULL) { return_error = BR_FAILED_REPLY; @@ -1604,6 +1683,10 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_binder_get_ref_failed; } + if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) { + return_error = BR_FAILED_REPLY; + goto err_binder_get_ref_failed; + } if (ref->node->proc == target_proc) { if (fp->type == BINDER_TYPE_HANDLE) fp->type = BINDER_TYPE_BINDER; @@ -1663,6 +1746,11 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_fget_failed; } + if (security_binder_transfer_file(proc->tsk, target_proc->tsk, file) < 0) { + fput(file); + return_error = BR_FAILED_REPLY; + goto err_get_unused_fd_failed; + } target_fd = task_get_unused_fd_flags(target_proc, O_CLOEXEC); if (target_fd < 0) { fput(file); @@ -1706,8 +1794,16 @@ static void binder_transaction(struct binder_proc *proc, list_add_tail(&t->work.entry, target_list); tcomplete->type = BINDER_WORK_TRANSACTION_COMPLETE; list_add_tail(&tcomplete->entry, &thread->todo); - if (target_wait) - wake_up_interruptible(target_wait); + if (target_wait) { + if (reply || !(t->flags & TF_ONE_WAY)) { + preempt_disable(); + wake_up_interruptible_sync(target_wait); + sched_preempt_enable_no_resched(); + } + else { + wake_up_interruptible(target_wait); + } + } return; err_get_unused_fd_failed: @@ -1766,7 +1862,7 @@ int binder_thread_write(struct binder_proc *proc, void __user *end = buffer + size; while (ptr < end && thread->return_error == BR_OK) { - if (get_user(cmd, (uint32_t __user *)ptr)) + if (get_user_preempt_disabled(cmd, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); trace_binder_command(cmd); @@ -1784,7 +1880,7 @@ int binder_thread_write(struct binder_proc *proc, struct binder_ref *ref; const char *debug_string; - if (get_user(target, (uint32_t __user *)ptr)) + if (get_user_preempt_disabled(target, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); if (target == 0 && binder_context_mgr_node && @@ -1843,10 +1939,10 @@ int binder_thread_write(struct binder_proc *proc, binder_uintptr_t cookie; struct binder_node *node; - if (get_user(node_ptr, (binder_uintptr_t __user *)ptr)) + if (get_user_preempt_disabled(node_ptr, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); - if (get_user(cookie, (binder_uintptr_t __user *)ptr)) + if (get_user_preempt_disabled(cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); node = binder_get_node(proc, node_ptr); @@ -1904,7 +2000,7 @@ int binder_thread_write(struct binder_proc *proc, binder_uintptr_t data_ptr; struct binder_buffer *buffer; - if (get_user(data_ptr, (binder_uintptr_t __user *)ptr)) + if (get_user_preempt_disabled(data_ptr, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); @@ -1946,7 +2042,7 @@ int binder_thread_write(struct binder_proc *proc, case BC_REPLY: { struct binder_transaction_data tr; - if (copy_from_user(&tr, ptr, sizeof(tr))) + if (copy_from_user_preempt_disabled(&tr, ptr, sizeof(tr))) return -EFAULT; ptr += sizeof(tr); binder_transaction(proc, thread, &tr, cmd == BC_REPLY); @@ -1996,10 +2092,10 @@ int binder_thread_write(struct binder_proc *proc, struct binder_ref *ref; struct binder_ref_death *death; - if (get_user(target, (uint32_t __user *)ptr)) + if (get_user_preempt_disabled(target, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); - if (get_user(cookie, (binder_uintptr_t __user *)ptr)) + if (get_user_preempt_disabled(cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); ref = binder_get_ref(proc, target, false); @@ -2028,7 +2124,7 @@ int binder_thread_write(struct binder_proc *proc, proc->pid, thread->pid); break; } - death = kzalloc(sizeof(*death), GFP_KERNEL); + death = kzalloc_preempt_disabled(sizeof(*death)); if (death == NULL) { thread->return_error = BR_ERROR; binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, @@ -2083,10 +2179,10 @@ int binder_thread_write(struct binder_proc *proc, binder_uintptr_t cookie; struct binder_ref_death *death = NULL; - if (get_user(cookie, (binder_uintptr_t __user *)ptr)) + if (get_user_preempt_disabled(cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; - ptr += sizeof(void *); + ptr += sizeof(cookie); list_for_each_entry(w, &proc->delivered_death, entry) { struct binder_ref_death *tmp_death = container_of(w, struct binder_ref_death, work); @@ -2163,7 +2259,7 @@ static int binder_thread_read(struct binder_proc *proc, int wait_for_proc_work; if (*consumed == 0) { - if (put_user(BR_NOOP, (uint32_t __user *)ptr)) + if (put_user_preempt_disabled(BR_NOOP, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); } @@ -2174,7 +2270,7 @@ static int binder_thread_read(struct binder_proc *proc, if (thread->return_error != BR_OK && ptr < end) { if (thread->return_error2 != BR_OK) { - if (put_user(thread->return_error2, (uint32_t __user *)ptr)) + if (put_user_preempt_disabled(thread->return_error2, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); binder_stat_br(proc, thread, thread->return_error2); @@ -2182,7 +2278,7 @@ static int binder_thread_read(struct binder_proc *proc, goto done; thread->return_error2 = BR_OK; } - if (put_user(thread->return_error, (uint32_t __user *)ptr)) + if (put_user_preempt_disabled(thread->return_error, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); binder_stat_br(proc, thread, thread->return_error); @@ -2197,7 +2293,6 @@ static int binder_thread_read(struct binder_proc *proc, binder_unlock(__func__); #if defined(VENDOR_EDIT) && defined(CONFIG_CGROUP_SCHED) - skip_cfs_throttle(0); #endif trace_binder_wait_for_work(wait_for_proc_work, @@ -2226,7 +2321,6 @@ static int binder_thread_read(struct binder_proc *proc, } #if defined(VENDOR_EDIT) && defined(CONFIG_CGROUP_SCHED) - skip_cfs_throttle(1); #endif binder_lock(__func__); @@ -2266,7 +2360,7 @@ static int binder_thread_read(struct binder_proc *proc, } break; case BINDER_WORK_TRANSACTION_COMPLETE: { cmd = BR_TRANSACTION_COMPLETE; - if (put_user(cmd, (uint32_t __user *)ptr)) + if (put_user_preempt_disabled(cmd, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); @@ -2308,14 +2402,14 @@ static int binder_thread_read(struct binder_proc *proc, node->has_weak_ref = 0; } if (cmd != BR_NOOP) { - if (put_user(cmd, (uint32_t __user *)ptr)) + if (put_user_preempt_disabled(cmd, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); - if (put_user(node->ptr, + if (put_user_preempt_disabled(node->ptr, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); - if (put_user(node->cookie, + if (put_user_preempt_disabled(node->cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); @@ -2359,10 +2453,10 @@ static int binder_thread_read(struct binder_proc *proc, cmd = BR_CLEAR_DEATH_NOTIFICATION_DONE; else cmd = BR_DEAD_BINDER; - if (put_user(cmd, (uint32_t __user *)ptr)) + if (put_user_preempt_disabled(cmd, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); - if (put_user(death->cookie, + if (put_user_preempt_disabled(death->cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); @@ -2430,10 +2524,10 @@ static int binder_thread_read(struct binder_proc *proc, ALIGN(t->buffer->data_size, sizeof(void *)); - if (put_user(cmd, (uint32_t __user *)ptr)) + if (put_user_preempt_disabled(cmd, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); - if (copy_to_user(ptr, &tr, sizeof(tr))) + if (copy_to_user_preempt_disabled(ptr, &tr, sizeof(tr))) return -EFAULT; ptr += sizeof(tr); @@ -2475,7 +2569,7 @@ static int binder_thread_read(struct binder_proc *proc, binder_debug(BINDER_DEBUG_THREADS, "%d:%d BR_SPAWN_LOOPER\n", proc->pid, thread->pid); - if (put_user(BR_SPAWN_LOOPER, (uint32_t __user *)buffer)) + if (put_user_preempt_disabled(BR_SPAWN_LOOPER, (uint32_t __user *)buffer)) return -EFAULT; binder_stat_br(proc, thread, BR_SPAWN_LOOPER); } @@ -2550,7 +2644,7 @@ static struct binder_thread *binder_get_thread(struct binder_proc *proc) break; } if (*p == NULL) { - thread = kzalloc(sizeof(*thread), GFP_KERNEL); + thread = kzalloc_preempt_disabled(sizeof(*thread)); if (thread == NULL) return NULL; binder_stats_created(BINDER_STAT_THREAD); @@ -2654,7 +2748,7 @@ static int binder_ioctl_write_read(struct file *filp, ret = -EINVAL; goto out; } - if (copy_from_user(&bwr, ubuf, sizeof(bwr))) { + if (copy_from_user_preempt_disabled(&bwr, ubuf, sizeof(bwr))) { ret = -EFAULT; goto out; } @@ -2672,7 +2766,7 @@ static int binder_ioctl_write_read(struct file *filp, trace_binder_write_done(ret); if (ret < 0) { bwr.read_consumed = 0; - if (copy_to_user(ubuf, &bwr, sizeof(bwr))) + if (copy_to_user_preempt_disabled(ubuf, &bwr, sizeof(bwr))) ret = -EFAULT; goto out; } @@ -2686,7 +2780,7 @@ static int binder_ioctl_write_read(struct file *filp, if (!list_empty(&proc->todo)) wake_up_interruptible(&proc->wait); if (ret < 0) { - if (copy_to_user(ubuf, &bwr, sizeof(bwr))) + if (copy_to_user_preempt_disabled(ubuf, &bwr, sizeof(bwr))) ret = -EFAULT; goto out; } @@ -2696,7 +2790,7 @@ static int binder_ioctl_write_read(struct file *filp, proc->pid, thread->pid, (u64)bwr.write_consumed, (u64)bwr.write_size, (u64)bwr.read_consumed, (u64)bwr.read_size); - if (copy_to_user(ubuf, &bwr, sizeof(bwr))) { + if (copy_to_user_preempt_disabled(ubuf, &bwr, sizeof(bwr))) { ret = -EFAULT; goto out; } @@ -2715,6 +2809,9 @@ static int binder_ioctl_set_ctx_mgr(struct file *filp) ret = -EBUSY; goto out; } + ret = security_binder_set_context_mgr(proc->tsk); + if (ret < 0) + goto out; if (uid_valid(binder_context_mgr_uid)) { if (!uid_eq(binder_context_mgr_uid, curr_euid)) { pr_err("BINDER_SET_CONTEXT_MGR bad uid %d != %d\n", @@ -2758,7 +2855,6 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) goto err_unlocked; #if defined(VENDOR_EDIT) && defined(CONFIG_CGROUP_SCHED) - skip_cfs_throttle(1); #endif binder_lock(__func__); thread = binder_get_thread(proc); @@ -2774,7 +2870,7 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) goto err; break; case BINDER_SET_MAX_THREADS: - if (copy_from_user(&proc->max_threads, ubuf, sizeof(proc->max_threads))) { + if (copy_from_user_preempt_disabled(&proc->max_threads, ubuf, sizeof(proc->max_threads))) { ret = -EINVAL; goto err; } @@ -2797,7 +2893,8 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) ret = -EINVAL; goto err; } - if (put_user(BINDER_CURRENT_PROTOCOL_VERSION, + + if (put_user_preempt_disabled(BINDER_CURRENT_PROTOCOL_VERSION, &ver->protocol_version)) { ret = -EINVAL; goto err; @@ -2814,7 +2911,6 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) thread->looper &= ~BINDER_LOOPER_STATE_NEED_RETURN; binder_unlock(__func__); #if defined(VENDOR_EDIT) && defined(CONFIG_CGROUP_SCHED) - skip_cfs_throttle(0); #endif wait_event_interruptible(binder_user_error_wait, binder_stop_on_user_error < 2); if (ret && ret != -ERESTARTSYS) @@ -2863,6 +2959,7 @@ static struct vm_operations_struct binder_vm_ops = { static int binder_mmap(struct file *filp, struct vm_area_struct *vma) { int ret; + struct vm_struct *area; struct binder_proc *proc = filp->private_data; const char *failure_string; @@ -2923,7 +3020,11 @@ static int binder_mmap(struct file *filp, struct vm_area_struct *vma) vma->vm_ops = &binder_vm_ops; vma->vm_private_data = proc; - if (binder_update_page_range(proc, 1, proc->buffer, proc->buffer + PAGE_SIZE, vma)) { + /* binder_update_page_range assumes preemption is disabled */ + preempt_disable(); + ret = binder_update_page_range(proc, 1, proc->buffer, proc->buffer + PAGE_SIZE, vma); + preempt_enable_no_resched(); + if (ret) { ret = -ENOMEM; failure_string = "alloc small buf"; goto err_alloc_small_buf_failed; @@ -3193,8 +3294,12 @@ static void binder_deferred_func(struct work_struct *work) int defer; do { - binder_lock(__func__); + trace_binder_lock(__func__); + mutex_lock(&binder_main_lock); + trace_binder_locked(__func__); + mutex_lock(&binder_deferred_lock); + preempt_disable(); if (!hlist_empty(&binder_deferred_list)) { proc = hlist_entry(binder_deferred_list.first, struct binder_proc, deferred_work_node); @@ -3220,7 +3325,9 @@ static void binder_deferred_func(struct work_struct *work) if (defer & BINDER_DEFERRED_RELEASE) binder_deferred_release(proc); /* frees proc */ - binder_unlock(__func__); + trace_binder_unlock(__func__); + mutex_unlock(&binder_main_lock); + preempt_enable_no_resched(); if (files) put_files_struct(files); } while (proc); diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index d5963ed5e5b13..821b91af9207c 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -87,7 +87,7 @@ static ssize_t __ref store_online(struct device *dev, } static DEVICE_ATTR(online, 0644, show_online, store_online); -static void __cpuinit register_cpu_control(struct cpu *cpu) +static void register_cpu_control(struct cpu *cpu) { device_create_file(&cpu->dev, &dev_attr_online); } @@ -465,7 +465,7 @@ static int cpu_uevent(struct device *dev, struct kobj_uevent_env *env) * * Initialize and register the CPU device. */ -int __cpuinit register_cpu(struct cpu *cpu, int num) +int register_cpu(struct cpu *cpu, int num) { int error; diff --git a/drivers/base/topology.c b/drivers/base/topology.c index 1eeb5f0048810..e4306a834b2e1 100644 --- a/drivers/base/topology.c +++ b/drivers/base/topology.c @@ -142,21 +142,21 @@ static struct attribute_group topology_attr_group = { }; /* Add/Remove cpu_topology interface for CPU device */ -static int __cpuinit topology_add_dev(unsigned int cpu) +static int topology_add_dev(unsigned int cpu) { struct device *dev = get_cpu_device(cpu); return sysfs_create_group(&dev->kobj, &topology_attr_group); } -static void __cpuinit topology_remove_dev(unsigned int cpu) +static void topology_remove_dev(unsigned int cpu) { struct device *dev = get_cpu_device(cpu); sysfs_remove_group(&dev->kobj, &topology_attr_group); } -static int __cpuinit topology_cpu_callback(struct notifier_block *nfb, +static int topology_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; @@ -177,7 +177,7 @@ static int __cpuinit topology_cpu_callback(struct notifier_block *nfb, return notifier_from_errno(rc); } -static int __cpuinit topology_sysfs_init(void) +static int topology_sysfs_init(void) { int cpu; int rc = 0; diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 1bdf38aab486f..dccb7e0c97e11 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -268,7 +268,7 @@ static int arch_timer_set_next_event_phys_mem(unsigned long evt, return 0; } -static void __cpuinit __arch_timer_setup(unsigned type, +static void __arch_timer_setup(unsigned type, struct clock_event_device *clk) { clk->features = CLOCK_EVT_FEAT_ONESHOT; @@ -499,7 +499,7 @@ static void __init arch_counter_register(unsigned type) sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate); } -static void __cpuinit arch_timer_stop(struct clock_event_device *clk) +static void arch_timer_stop(struct clock_event_device *clk) { pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id()); @@ -515,7 +515,7 @@ static void __cpuinit arch_timer_stop(struct clock_event_device *clk) clk->set_mode(CLOCK_EVT_MODE_UNUSED, clk); } -static int __cpuinit arch_timer_cpu_notify(struct notifier_block *self, +static int arch_timer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { /* @@ -534,7 +534,7 @@ static int __cpuinit arch_timer_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block arch_timer_cpu_nb __cpuinitdata = { +static struct notifier_block arch_timer_cpu_nb = { .notifier_call = arch_timer_cpu_notify, }; diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c index 3dfa3e5e3705e..7b98e6472631e 100644 --- a/drivers/clocksource/exynos_mct.c +++ b/drivers/clocksource/exynos_mct.c @@ -412,7 +412,7 @@ static struct irqaction mct_tick1_event_irq = { .handler = exynos4_mct_tick_isr, }; -static int __cpuinit exynos4_local_timer_setup(struct clock_event_device *evt) +static int exynos4_local_timer_setup(struct clock_event_device *evt) { struct mct_clock_event_device *mevt; unsigned int cpu = smp_processor_id(); @@ -465,7 +465,7 @@ static void exynos4_local_timer_stop(struct clock_event_device *evt) disable_percpu_irq(mct_irqs[MCT_L0_IRQ]); } -static struct local_timer_ops exynos4_mct_tick_ops __cpuinitdata = { +static struct local_timer_ops exynos4_mct_tick_ops = { .setup = exynos4_local_timer_setup, .stop = exynos4_local_timer_stop, }; diff --git a/drivers/clocksource/metag_generic.c b/drivers/clocksource/metag_generic.c index ade7513a11d1d..05ccb99acf3ef 100644 --- a/drivers/clocksource/metag_generic.c +++ b/drivers/clocksource/metag_generic.c @@ -109,7 +109,7 @@ unsigned long long sched_clock(void) return ticks << HARDWARE_TO_NS_SHIFT; } -static void __cpuinit arch_timer_setup(unsigned int cpu) +static void arch_timer_setup(unsigned int cpu) { unsigned int txdivtime; struct clock_event_device *clk = &per_cpu(local_clockevent, cpu); @@ -154,7 +154,7 @@ static void __cpuinit arch_timer_setup(unsigned int cpu) } } -static int __cpuinit arch_timer_cpu_notify(struct notifier_block *self, +static int arch_timer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { int cpu = (long)hcpu; @@ -169,7 +169,7 @@ static int __cpuinit arch_timer_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata arch_timer_cpu_nb = { +static struct notifier_block arch_timer_cpu_nb = { .notifier_call = arch_timer_cpu_notify, }; diff --git a/drivers/clocksource/time-armada-370-xp.c b/drivers/clocksource/time-armada-370-xp.c index efdca3263afef..1b04b7e1d39b9 100644 --- a/drivers/clocksource/time-armada-370-xp.c +++ b/drivers/clocksource/time-armada-370-xp.c @@ -167,7 +167,7 @@ static irqreturn_t armada_370_xp_timer_interrupt(int irq, void *dev_id) /* * Setup the local clock events for a CPU. */ -static int __cpuinit armada_370_xp_timer_setup(struct clock_event_device *evt) +static int armada_370_xp_timer_setup(struct clock_event_device *evt) { u32 u; int cpu = smp_processor_id(); @@ -205,7 +205,7 @@ static void armada_370_xp_timer_stop(struct clock_event_device *evt) disable_percpu_irq(evt->irq); } -static struct local_timer_ops armada_370_xp_local_timer_ops __cpuinitdata = { +static struct local_timer_ops armada_370_xp_local_timer_ops = { .setup = armada_370_xp_timer_setup, .stop = armada_370_xp_timer_stop, }; diff --git a/drivers/clocksource/timer-marco.c b/drivers/clocksource/timer-marco.c index e5dc9129ca263..62876baa3ab9c 100644 --- a/drivers/clocksource/timer-marco.c +++ b/drivers/clocksource/timer-marco.c @@ -184,7 +184,7 @@ static struct irqaction sirfsoc_timer1_irq = { .handler = sirfsoc_timer_interrupt, }; -static int __cpuinit sirfsoc_local_timer_setup(struct clock_event_device *ce) +static int sirfsoc_local_timer_setup(struct clock_event_device *ce) { /* Use existing clock_event for cpu 0 */ if (!smp_processor_id()) @@ -216,7 +216,7 @@ static void sirfsoc_local_timer_stop(struct clock_event_device *ce) remove_irq(sirfsoc_timer1_irq.irq, &sirfsoc_timer1_irq); } -static struct local_timer_ops sirfsoc_local_timer_ops __cpuinitdata = { +static struct local_timer_ops sirfsoc_local_timer_ops = { .setup = sirfsoc_local_timer_setup, .stop = sirfsoc_local_timer_stop, }; diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index 29afa37aa4c12..2b2030fe895c7 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -118,6 +118,15 @@ config CPU_FREQ_DEFAULT_GOV_INTERACTIVE loading your cpufreq low-level hardware driver, using the 'interactive' governor for latency-sensitive workloads. +config CPU_FREQ_DEFAULT_GOV_SCHED + bool "sched" + select CPU_FREQ_GOV_SCHED + select CPU_FREQ_GOV_PERFORMANCE + help + Use the CPUfreq governor 'sched' as default. This scales + cpu frequency from the scheduler as per-entity load tracking + statistics are updated. + endchoice config CPU_FREQ_GOV_PERFORMANCE @@ -216,6 +225,21 @@ config CPU_FREQ_GOV_CONSERVATIVE If in doubt, say N. +config CPU_FREQ_GOV_SCHED + tristate "'sched' cpufreq governor" + depends on CPU_FREQ + select CPU_FREQ_GOV_COMMON + help + 'sched' - this governor scales cpu frequency from the + scheduler as a function of cpu capacity utilization. It does + not evaluate utilization on a periodic basis (as ondemand + does) but instead is invoked from the completely fair + scheduler when updating per-entity load tracking statistics. + Latency to respond to changes in load is improved over polling + governors due to its event-driven design. + + If in doubt, say N. + config CPU_BOOST tristate "Event base short term CPU freq boost" depends on CPU_FREQ diff --git a/drivers/cpufreq/cpu-boost.c b/drivers/cpufreq/cpu-boost.c index b5b3fbf5d1c09..545d484b17596 100644 --- a/drivers/cpufreq/cpu-boost.c +++ b/drivers/cpufreq/cpu-boost.c @@ -48,11 +48,6 @@ static struct work_struct input_boost_work; static unsigned int boost_ms; module_param(boost_ms, uint, 0644); -#ifdef VENDOR_EDIT -extern unsigned int sysctl_thermal_aware_scheduling; -module_param(sysctl_thermal_aware_scheduling, uint, 0644); -#endif - static unsigned int sync_threshold; module_param(sync_threshold, uint, 0644); diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index a0913ab299603..24def50615b76 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -118,6 +118,12 @@ bool have_governor_per_policy(void) } EXPORT_SYMBOL_GPL(have_governor_per_policy); +bool cpufreq_driver_is_slow(void) +{ + return !(cpufreq_driver->flags & CPUFREQ_DRIVER_FAST); +} +EXPORT_SYMBOL_GPL(cpufreq_driver_is_slow); + struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) { if (have_governor_per_policy()) @@ -275,6 +281,50 @@ static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci) } #endif +/********************************************************************* + * FREQUENCY INVARIANT CPU CAPACITY * + *********************************************************************/ + +static DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE; +static DEFINE_PER_CPU(unsigned long, max_freq_scale) = SCHED_CAPACITY_SCALE; + +static void +scale_freq_capacity(struct cpufreq_policy *policy, struct cpufreq_freqs *freqs) +{ + unsigned long cur = freqs ? freqs->new : policy->cur; + unsigned long scale = (cur << SCHED_CAPACITY_SHIFT) / policy->max; + struct cpufreq_cpuinfo *cpuinfo = &policy->cpuinfo; + int cpu; + + pr_debug("cpus %*pbl cur/cur max freq %lu/%u kHz freq scale %lu\n", + cpumask_pr_args(policy->cpus), cur, policy->max, scale); + + for_each_cpu(cpu, policy->cpus) + per_cpu(freq_scale, cpu) = scale; + + if (freqs) + return; + + scale = (policy->max << SCHED_CAPACITY_SHIFT) / cpuinfo->max_freq; + + pr_debug("cpus %*pbl cur max/max freq %u/%u kHz max freq scale %lu\n", + cpumask_pr_args(policy->cpus), policy->max, cpuinfo->max_freq, + scale); + + for_each_cpu(cpu, policy->cpus) + per_cpu(max_freq_scale, cpu) = scale; +} + +unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu) +{ + return per_cpu(freq_scale, cpu); +} + +unsigned long cpufreq_scale_max_freq_capacity(int cpu) +{ + return per_cpu(max_freq_scale, cpu); +} + static void __cpufreq_notify_transition(struct cpufreq_policy *policy, struct cpufreq_freqs *freqs, unsigned int state) { @@ -337,7 +387,6 @@ void cpufreq_notify_transition(struct cpufreq_policy *policy, } EXPORT_SYMBOL_GPL(cpufreq_notify_transition); - /********************************************************************* * SYSFS INTERFACE * *********************************************************************/ @@ -2036,6 +2085,8 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_NOTIFY, new_policy); + scale_freq_capacity(new_policy, NULL); + policy->min = new_policy->min; policy->max = new_policy->max; trace_cpu_frequency_limits(policy->max, policy->min, policy->cpu); diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c index 59990ac20f22c..ba888b776d775 100644 --- a/drivers/cpufreq/cpufreq_interactive.c +++ b/drivers/cpufreq/cpufreq_interactive.c @@ -36,6 +36,11 @@ #define CREATE_TRACE_POINTS #include +#ifndef TJK_HMP +ATOMIC_NOTIFIER_HEAD(migration_notifier_head); +ATOMIC_NOTIFIER_HEAD(load_alert_notifier_head); +#endif + struct cpufreq_interactive_policyinfo { struct timer_list policy_timer; struct timer_list policy_slack_timer; diff --git a/drivers/cpufreq/cpufreq_persistent_stats.c b/drivers/cpufreq/cpufreq_persistent_stats.c index 0b2ded84e17b8..dbd86f30a49f1 100644 --- a/drivers/cpufreq/cpufreq_persistent_stats.c +++ b/drivers/cpufreq/cpufreq_persistent_stats.c @@ -268,7 +268,7 @@ static int cpufreq_stat_notifier_trans(struct notifier_block *nb, return 0; } -static int __cpuinit cpufreq_stat_cpu_callback(struct notifier_block *nfb, +static int cpufreq_stat_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { diff --git a/drivers/cpuidle/lpm-levels.c b/drivers/cpuidle/lpm-levels.c index ad0d407f92bac..e93b587102aba 100644 --- a/drivers/cpuidle/lpm-levels.c +++ b/drivers/cpuidle/lpm-levels.c @@ -696,7 +696,7 @@ static int lpm_cpuidle_enter(struct cpuidle_device *dev, } pwr_params = &cluster->cpu->levels[idx].pwr; - sched_set_cpu_cstate(smp_processor_id(), idx + 1, + sched_set_cpu_cstate(smp_processor_id(), idx + 1, &drv->states[idx+1], pwr_params->energy_overhead, pwr_params->latency_us); cpu_prepare(cluster, idx, true); @@ -710,7 +710,7 @@ static int lpm_cpuidle_enter(struct cpuidle_device *dev, cluster_unprepare(cluster, cpumask, idx, true); cpu_unprepare(cluster, idx, true); - sched_set_cpu_cstate(smp_processor_id(), 0, 0, 0); + sched_set_cpu_cstate(smp_processor_id(), 0, &drv->states[0], 0, 0); time = ktime_to_ns(ktime_get()) - time; do_div(time, 1000); diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index 0cf25101b3c3e..9f8ac97e78be9 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -195,7 +195,7 @@ struct tjmax { int tjmax; }; -static const struct tjmax __cpuinitconst tjmax_table[] = { +static const struct tjmax tjmax_table[] = { { "CPU 230", 100000 }, /* Model 0x1c, stepping 2 */ { "CPU 330", 125000 }, /* Model 0x1c, stepping 2 */ { "CPU CE4110", 110000 }, /* Model 0x1c, stepping 10 Sodaville */ @@ -211,7 +211,7 @@ struct tjmax_model { #define ANY 0xff -static const struct tjmax_model __cpuinitconst tjmax_model_table[] = { +static const struct tjmax_model tjmax_model_table[] = { { 0x1c, 10, 100000 }, /* D4xx, K4xx, N4xx, D5xx, K5xx, N5xx */ { 0x1c, ANY, 90000 }, /* Z5xx, N2xx, possibly others * Note: Also matches 230 and 330, @@ -226,7 +226,7 @@ static const struct tjmax_model __cpuinitconst tjmax_model_table[] = { { 0x36, ANY, 100000 }, /* Atom Cedar Trail/Cedarview (N2xxx, D2xxx) */ }; -static int __cpuinit adjust_tjmax(struct cpuinfo_x86 *c, u32 id, +static int adjust_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev) { /* The 100C is default for both mobile and non mobile CPUs */ @@ -317,7 +317,7 @@ static int __cpuinit adjust_tjmax(struct cpuinfo_x86 *c, u32 id, return tjmax; } -static int __cpuinit get_tjmax(struct cpuinfo_x86 *c, u32 id, +static int get_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev) { int err; @@ -367,7 +367,7 @@ static int create_name_attr(struct platform_data *pdata, return device_create_file(dev, &pdata->name_attr); } -static int __cpuinit create_core_attrs(struct temp_data *tdata, +static int create_core_attrs(struct temp_data *tdata, struct device *dev, int attr_no) { int err, i; @@ -401,7 +401,7 @@ static int __cpuinit create_core_attrs(struct temp_data *tdata, } -static int __cpuinit chk_ucode_version(unsigned int cpu) +static int chk_ucode_version(unsigned int cpu) { struct cpuinfo_x86 *c = &cpu_data(cpu); @@ -417,7 +417,7 @@ static int __cpuinit chk_ucode_version(unsigned int cpu) return 0; } -static struct platform_device __cpuinit *coretemp_get_pdev(unsigned int cpu) +static struct platform_device *coretemp_get_pdev(unsigned int cpu) { u16 phys_proc_id = TO_PHYS_ID(cpu); struct pdev_entry *p; @@ -434,7 +434,7 @@ static struct platform_device __cpuinit *coretemp_get_pdev(unsigned int cpu) return NULL; } -static struct temp_data __cpuinit *init_temp_data(unsigned int cpu, +static struct temp_data *init_temp_data(unsigned int cpu, int pkg_flag) { struct temp_data *tdata; @@ -453,7 +453,7 @@ static struct temp_data __cpuinit *init_temp_data(unsigned int cpu, return tdata; } -static int __cpuinit create_core_data(struct platform_device *pdev, +static int create_core_data(struct platform_device *pdev, unsigned int cpu, int pkg_flag) { struct temp_data *tdata; @@ -524,7 +524,7 @@ static int __cpuinit create_core_data(struct platform_device *pdev, return err; } -static void __cpuinit coretemp_add_core(unsigned int cpu, int pkg_flag) +static void coretemp_add_core(unsigned int cpu, int pkg_flag) { struct platform_device *pdev = coretemp_get_pdev(cpu); int err; @@ -609,7 +609,7 @@ static struct platform_driver coretemp_driver = { .remove = coretemp_remove, }; -static int __cpuinit coretemp_device_add(unsigned int cpu) +static int coretemp_device_add(unsigned int cpu) { int err; struct platform_device *pdev; @@ -653,7 +653,7 @@ static int __cpuinit coretemp_device_add(unsigned int cpu) return err; } -static void __cpuinit coretemp_device_remove(unsigned int cpu) +static void coretemp_device_remove(unsigned int cpu) { struct pdev_entry *p, *n; u16 phys_proc_id = TO_PHYS_ID(cpu); @@ -669,7 +669,7 @@ static void __cpuinit coretemp_device_remove(unsigned int cpu) mutex_unlock(&pdev_list_mutex); } -static bool __cpuinit is_any_core_online(struct platform_data *pdata) +static bool is_any_core_online(struct platform_data *pdata) { int i; @@ -683,7 +683,7 @@ static bool __cpuinit is_any_core_online(struct platform_data *pdata) return false; } -static void __cpuinit get_core_online(unsigned int cpu) +static void get_core_online(unsigned int cpu) { struct cpuinfo_x86 *c = &cpu_data(cpu); struct platform_device *pdev = coretemp_get_pdev(cpu); @@ -725,7 +725,7 @@ static void __cpuinit get_core_online(unsigned int cpu) coretemp_add_core(cpu, 0); } -static void __cpuinit put_core_offline(unsigned int cpu) +static void put_core_offline(unsigned int cpu) { int i, indx; struct platform_data *pdata; @@ -773,7 +773,7 @@ static void __cpuinit put_core_offline(unsigned int cpu) coretemp_device_remove(cpu); } -static int __cpuinit coretemp_cpu_callback(struct notifier_block *nfb, +static int coretemp_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long) hcpu; diff --git a/drivers/hwmon/via-cputemp.c b/drivers/hwmon/via-cputemp.c index 76f157b568ed7..65edf7da86edb 100644 --- a/drivers/hwmon/via-cputemp.c +++ b/drivers/hwmon/via-cputemp.c @@ -221,7 +221,7 @@ struct pdev_entry { static LIST_HEAD(pdev_list); static DEFINE_MUTEX(pdev_list_mutex); -static int __cpuinit via_cputemp_device_add(unsigned int cpu) +static int via_cputemp_device_add(unsigned int cpu) { int err; struct platform_device *pdev; @@ -262,7 +262,7 @@ static int __cpuinit via_cputemp_device_add(unsigned int cpu) return err; } -static void __cpuinit via_cputemp_device_remove(unsigned int cpu) +static void via_cputemp_device_remove(unsigned int cpu) { struct pdev_entry *p; @@ -279,7 +279,7 @@ static void __cpuinit via_cputemp_device_remove(unsigned int cpu) mutex_unlock(&pdev_list_mutex); } -static int __cpuinit via_cputemp_cpu_callback(struct notifier_block *nfb, +static int via_cputemp_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long) hcpu; diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index f26ff9e9a55fd..e5ca0fbd41028 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -630,7 +630,7 @@ static void __init gic_dist_init(struct gic_chip_data *gic) mb(); } -static void __cpuinit gic_cpu_init(struct gic_chip_data *gic) +static void gic_cpu_init(struct gic_chip_data *gic) { void __iomem *dist_base = gic_data_dist_base(gic); void __iomem *base = gic_data_cpu_base(gic); @@ -990,7 +990,7 @@ static int gic_irq_domain_xlate(struct irq_domain *d, } #ifdef CONFIG_SMP -static int __cpuinit gic_secondary_init(struct notifier_block *nfb, +static int gic_secondary_init(struct notifier_block *nfb, unsigned long action, void *hcpu) { if (action == CPU_STARTING || action == CPU_STARTING_FROZEN) @@ -1002,7 +1002,7 @@ static int __cpuinit gic_secondary_init(struct notifier_block *nfb, * Notifier for enabling the GIC CPU interface. Set an arbitrarily high * priority because the GIC needs to be up before the ARM generic timers. */ -static struct notifier_block __cpuinitdata gic_cpu_notifier = { +static struct notifier_block gic_cpu_notifier = { .notifier_call = gic_secondary_init, .priority = 100, }; diff --git a/drivers/oprofile/timer_int.c b/drivers/oprofile/timer_int.c index 93404f72dfa81..235659af2c425 100644 --- a/drivers/oprofile/timer_int.c +++ b/drivers/oprofile/timer_int.c @@ -74,7 +74,7 @@ static void oprofile_hrtimer_stop(void) put_online_cpus(); } -static int __cpuinit oprofile_cpu_notify(struct notifier_block *self, +static int oprofile_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { long cpu = (long) hcpu; diff --git a/drivers/xen/xen-acpi-cpuhotplug.c b/drivers/xen/xen-acpi-cpuhotplug.c index 18c742bec91bf..b7d5d95d8abf9 100644 --- a/drivers/xen/xen-acpi-cpuhotplug.c +++ b/drivers/xen/xen-acpi-cpuhotplug.c @@ -89,7 +89,7 @@ static int xen_acpi_processor_enable(struct acpi_device *device) return 0; } -static int __cpuinit xen_acpi_processor_add(struct acpi_device *device) +static int xen_acpi_processor_add(struct acpi_device *device) { int ret; struct acpi_processor *pr; diff --git a/fs/exec.c b/fs/exec.c index 0ca8cbac20cd4..618d3ac29b368 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1570,11 +1570,6 @@ static int do_execve_common(const char *filename, if (retval < 0) goto out; - if (d_is_su(file->f_dentry) && capable(CAP_SYS_ADMIN)) { - current->flags |= PF_SU; - su_exec(); - } - /* execve succeeded */ current->fs->in_exec = 0; current->in_execve = 0; diff --git a/fs/namei.c b/fs/namei.c index bdfbb34dd137d..06942ce4f7e75 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -2026,16 +2026,6 @@ static int path_lookupat(int dfd, const char *name, } } - if (!err) { - struct super_block *sb = nd->inode->i_sb; - if (sb->s_flags & MS_RDONLY) { - if (d_is_su(nd->path.dentry) && !su_visible()) { - path_put(&nd->path); - err = -ENOENT; - } - } - } - if (base) fput(base); diff --git a/fs/open.c b/fs/open.c index 3dd51d4af04b5..da01069ad4d43 100644 --- a/fs/open.c +++ b/fs/open.c @@ -940,7 +940,6 @@ long do_sys_open(int dfd, const char __user *filename, int flags, umode_t mode) struct filename *tmp = getname(filename); int fd = PTR_ERR(tmp); - skip_cfs_throttle(1); if (!IS_ERR(tmp)) { fd = get_unused_fd_flags(flags); if (fd >= 0) { @@ -955,7 +954,6 @@ long do_sys_open(int dfd, const char __user *filename, int flags, umode_t mode) } putname(tmp); } - skip_cfs_throttle(0); return fd; } diff --git a/fs/proc/base.c b/fs/proc/base.c index d3410433ed49b..b5d1fedf486bb 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -1326,11 +1326,14 @@ static int sched_wake_up_idle_show(struct seq_file *m, void *v) if (!p) return -ESRCH; +#ifdef TJK_HMP seq_printf(m, "%d\n", sched_get_wake_up_idle(p)); put_task_struct(p); - return 0; +#else + return -EINVAL; +#endif } static ssize_t @@ -1358,9 +1361,13 @@ sched_wake_up_idle_write(struct file *file, const char __user *buf, if (!p) return -ESRCH; +#ifdef TJK_HMP err = sched_set_wake_up_idle(p, wake_up_idle); put_task_struct(p); +#else + err = -EINVAL; +#endif out: return err < 0 ? err : count; diff --git a/fs/readdir.c b/fs/readdir.c index b22bbbfabeeb5..d46eca8567a41 100644 --- a/fs/readdir.c +++ b/fs/readdir.c @@ -37,7 +37,6 @@ int iterate_dir(struct file *file, struct dir_context *ctx) res = -ENOENT; if (!IS_DEADDIR(inode)) { - ctx->romnt = (inode->i_sb->s_flags & MS_RDONLY); if (file->f_op->iterate) { ctx->pos = file->f_pos; res = file->f_op->iterate(file, ctx); @@ -54,14 +53,6 @@ int iterate_dir(struct file *file, struct dir_context *ctx) } EXPORT_SYMBOL(iterate_dir); -static bool hide_name(const char *name, int namlen) -{ - if (namlen == 2 && !memcmp(name, "su", 2)) - if (!su_visible()) - return true; - return false; -} - /* * Traditional linux readdir() handling.. * @@ -100,8 +91,6 @@ static int fillonedir(void * __buf, const char * name, int namlen, loff_t offset buf->result = -EOVERFLOW; return -EOVERFLOW; } - if (hide_name(name, namlen) && buf->ctx.romnt) - return 0; buf->result++; dirent = buf->dirent; if (!access_ok(VERIFY_WRITE, dirent, @@ -179,8 +168,6 @@ static int filldir(void * __buf, const char * name, int namlen, loff_t offset, buf->error = -EOVERFLOW; return -EOVERFLOW; } - if (hide_name(name, namlen) && buf->ctx.romnt) - return 0; dirent = buf->previous; if (dirent) { if (__put_user(offset, &dirent->d_off)) @@ -259,8 +246,6 @@ static int filldir64(void * __buf, const char * name, int namlen, loff_t offset, buf->error = -EINVAL; /* only used if we fail.. */ if (reclen > buf->count) return -EINVAL; - if (hide_name(name, namlen) && buf->ctx.romnt) - return 0; dirent = buf->previous; if (dirent) { if (__put_user(offset, &dirent->d_off)) diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h index d17784ea37ff3..344c19565813c 100644 --- a/include/asm-generic/percpu.h +++ b/include/asm-generic/percpu.h @@ -87,6 +87,9 @@ extern void setup_per_cpu_areas(void); #endif /* SMP */ +/* 3.18 backport */ +#define raw_cpu_ptr(ptr) __this_cpu_ptr(ptr) + #ifndef PER_CPU_BASE_SECTION #ifdef CONFIG_SMP #define PER_CPU_BASE_SECTION ".data..percpu" diff --git a/include/linux/cgroup_subsys.h b/include/linux/cgroup_subsys.h index e5e6b0dd41d9e..61fefd93fafe4 100644 --- a/include/linux/cgroup_subsys.h +++ b/include/linux/cgroup_subsys.h @@ -31,6 +31,12 @@ SUBSYS(cpuacct) /* */ +#if IS_SUBSYS_ENABLED(CONFIG_CGROUP_SCHEDTUNE) +SUBSYS(schedtune) +#endif + +/* */ + #if IS_SUBSYS_ENABLED(CONFIG_MEMCG) SUBSYS(mem_cgroup) #endif diff --git a/include/linux/cpu.h b/include/linux/cpu.h index 33f55c8c59d1f..70084db409f68 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -129,13 +129,13 @@ enum { /* Need to know about CPUs going up/down? */ #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) #define cpu_notifier(fn, pri) { \ - static struct notifier_block fn##_nb __cpuinitdata = \ + static struct notifier_block fn##_nb = \ { .notifier_call = fn, .priority = pri }; \ register_cpu_notifier(&fn##_nb); \ } #define __cpu_notifier(fn, pri) { \ - static struct notifier_block fn##_nb __cpuinitdata = \ + static struct notifier_block fn##_nb = \ { .notifier_call = fn, .priority = pri }; \ __register_cpu_notifier(&fn##_nb); \ } diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index d3c1819dc0d48..70ee2c9e2e690 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -141,6 +141,7 @@ u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy); int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu); int cpufreq_update_policy(unsigned int cpu); bool have_governor_per_policy(void); +bool cpufreq_driver_is_slow(void); struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy); #else static inline unsigned int cpufreq_get(unsigned int cpu) @@ -253,6 +254,23 @@ struct cpufreq_driver { */ #define CPUFREQ_ASYNC_NOTIFICATION (1 << 4) +/* + * Set by drivers which want cpufreq core to check if CPU is running at a + * frequency present in freq-table exposed by the driver. For these drivers if + * CPU is found running at an out of table freq, we will try to set it to a freq + * from the table. And if that fails, we will stop further boot process by + * issuing a BUG_ON(). + */ +#define CPUFREQ_NEED_INITIAL_FREQ_CHECK (1 << 5) + +/* + * Indicates that it is safe to call cpufreq_driver_target from + * non-interruptable context in scheduler hot paths. Drivers must + * opt-in to this flag, as the safe default is that they might sleep + * or be too slow for hot path use. + */ +#define CPUFREQ_DRIVER_FAST (1 << 6) + int cpufreq_register_driver(struct cpufreq_driver *driver_data); int cpufreq_unregister_driver(struct cpufreq_driver *driver_data); @@ -432,6 +450,9 @@ extern struct cpufreq_governor cpufreq_gov_conservative; #elif defined(CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE) extern struct cpufreq_governor cpufreq_gov_interactive; #define CPUFREQ_DEFAULT_GOVERNOR (&cpufreq_gov_interactive) +#elif defined(CONFIG_CPU_FREQ_DEFAULT_GOV_SCHED) +extern struct cpufreq_governor cpufreq_gov_sched; +#define CPUFREQ_DEFAULT_GOVERNOR (&cpufreq_gov_sched) #endif /********************************************************************* @@ -488,6 +509,19 @@ static inline int cpufreq_generic_exit(struct cpufreq_policy *policy) * CPUFREQ STATS * *********************************************************************/ +#ifdef CONFIG_CPU_FREQ_STAT + void acct_update_power(struct task_struct *p, cputime_t cputime); +#else + +static inline void acct_update_power(struct task_struct *p, cputime_t cputime) +{ +} + +#endif + +struct sched_domain; +unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu); +unsigned long cpufreq_scale_max_freq_capacity(int cpu); #endif /* _LINUX_CPUFREQ_H */ diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index d08e4d2a9b925..7abda3b903db5 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -22,6 +22,14 @@ typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t; */ #define cpumask_bits(maskp) ((maskp)->bits) +/** + * cpumask_pr_args - printf args to output a cpumask + * @maskp: cpumask to be printed + * + * Can be used to provide arguments for '%*pb[l]' when printing a cpumask. + */ +#define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp) + #if NR_CPUS == 1 #define nr_cpu_ids 1 #else diff --git a/include/linux/cputime.h b/include/linux/cputime.h new file mode 100644 index 0000000000000..f2eb2ee535cab --- /dev/null +++ b/include/linux/cputime.h @@ -0,0 +1,16 @@ +#ifndef __LINUX_CPUTIME_H +#define __LINUX_CPUTIME_H + +#include + +#ifndef cputime_to_nsecs +# define cputime_to_nsecs(__ct) \ + (cputime_to_usecs(__ct) * NSEC_PER_USEC) +#endif + +#ifndef nsecs_to_cputime +# define nsecs_to_cputime(__nsecs) \ + usecs_to_cputime((__nsecs) / NSEC_PER_USEC) +#endif + +#endif /* __LINUX_CPUTIME_H */ diff --git a/include/linux/dcache.h b/include/linux/dcache.h index f2c042f1cb81a..e91a63f63e462 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h @@ -413,13 +413,6 @@ static inline bool d_mountpoint(struct dentry *dentry) return dentry->d_flags & DCACHE_MOUNTED; } -static inline bool d_is_su(const struct dentry *dentry) -{ - return dentry && - dentry->d_name.len == 2 && - !memcmp(dentry->d_name.name, "su", 2); -} - extern int sysctl_vfs_cache_pressure; #endif /* __LINUX_DCACHE_H */ diff --git a/include/linux/fs.h b/include/linux/fs.h index 4682822c3fc91..9632f0bb43b6e 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1535,7 +1535,6 @@ typedef int (*filldir_t)(void *, const char *, int, loff_t, u64, unsigned); struct dir_context { const filldir_t actor; loff_t pos; - bool romnt; }; static inline bool dir_emit(struct dir_context *ctx, diff --git a/include/linux/init.h b/include/linux/init.h index 861814710d523..20d992b73d72e 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -94,12 +94,12 @@ #define __exit __section(.exit.text) __exitused __cold notrace /* Used for HOTPLUG_CPU */ -#define __cpuinit __section(.cpuinit.text) __cold notrace -#define __cpuinitdata __section(.cpuinit.data) -#define __cpuinitconst __constsection(.cpuinit.rodata) -#define __cpuexit __section(.cpuexit.text) __exitused __cold notrace -#define __cpuexitdata __section(.cpuexit.data) -#define __cpuexitconst __constsection(.cpuexit.rodata) +#define __cpuinit __cold +#define __cpuinitdata +#define __cpuinitconst +#define __cpuexit __exitused __cold +#define __cpuexitdata +#define __cpuexitconst /* Used for MEMORY_HOTPLUG */ #define __meminit __section(.meminit.text) __cold notrace diff --git a/include/linux/kthread.h b/include/linux/kthread.h index 7dcef3317689e..416d23513b348 100644 --- a/include/linux/kthread.h +++ b/include/linux/kthread.h @@ -75,6 +75,8 @@ struct kthread_work { kthread_work_func_t func; wait_queue_head_t done; struct kthread_worker *worker; + /* Number of canceling calls that are running at the moment. */ + int canceling; }; #define KTHREAD_WORKER_INIT(worker) { \ @@ -137,4 +139,6 @@ bool queue_kthread_work(struct kthread_worker *worker, void flush_kthread_work(struct kthread_work *work); void flush_kthread_worker(struct kthread_worker *worker); +bool kthread_cancel_work_sync(struct kthread_work *work); + #endif /* _LINUX_KTHREAD_H */ diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h index 4e2cbfa640b72..51f6783a2fef9 100644 --- a/include/linux/nodemask.h +++ b/include/linux/nodemask.h @@ -98,6 +98,23 @@ typedef struct { DECLARE_BITMAP(bits, MAX_NUMNODES); } nodemask_t; extern nodemask_t _unused_nodemask_arg_; +/** + * nodemask_pr_args - printf args to output a nodemask + * @maskp: nodemask to be printed + * + * Can be used to provide arguments for '%*pb[l]' when printing a nodemask. + */ +#define nodemask_pr_args(maskp) MAX_NUMNODES, (maskp)->bits + +/* + * The inline keyword gives the compiler room to decide to inline, or + * not inline a function as it sees best. However, as these functions + * are called in both __init and non-__init functions, if they are not + * inlined we will end up with a section mis-match error (of the type of + * freeable items not being freed). So we must use __always_inline here + * to fix the problem. If other functions in the future also end up in + * this situation they will also need to be annotated as __always_inline + */ #define node_set(node, dst) __node_set((node), &(dst)) static inline void __node_set(int node, volatile nodemask_t *dstp) { diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index b10868f51332e..54aad39a8a82e 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -826,7 +826,7 @@ static inline void perf_restore_debug_store(void) { } */ #define perf_cpu_notifier(fn) \ do { \ - static struct notifier_block fn##_nb __cpuinitdata = \ + static struct notifier_block fn##_nb = \ { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ unsigned long cpu = smp_processor_id(); \ unsigned long flags; \ diff --git a/include/linux/preempt.h b/include/linux/preempt.h index f5d4723cdb3d3..674ff1d7fd7f8 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -10,7 +10,7 @@ #include #include -#if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER) +#if 0 && (defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER)) extern void add_preempt_count(int val); extern void sub_preempt_count(int val); #else @@ -23,6 +23,10 @@ #define preempt_count() (current_thread_info()->preempt_count) +#ifndef TJK_HMP +#define set_preempt_need_resched() +#endif + #ifdef CONFIG_PREEMPT asmlinkage void preempt_schedule(void); @@ -126,6 +130,9 @@ do { \ #endif /* CONFIG_PREEMPT_COUNT */ +#define preempt_set_need_resched() +#define preempt_fold_need_resched() + #ifdef CONFIG_PREEMPT_NOTIFIERS struct preempt_notifier; diff --git a/include/linux/sched.h b/include/linux/sched.h index 53b9e2d0da438..b389a1292adf9 100755 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -3,6 +3,8 @@ #include +#include + struct sched_param { int sched_priority; @@ -16,19 +18,24 @@ struct sched_param { #include #include #include +#include #include #include #include #include #include #include +#include #include #include -#include +#include #include #include +#ifdef TJK_HMP +#include +#endif #include #include #include @@ -52,15 +59,10 @@ struct sched_param { #include #include #include +#include #include -int su_instances(void); -bool su_running(void); -bool su_visible(void); -void su_exec(void); -void su_exit(void); - #define SCHED_ATTR_SIZE_VER0 48 /* sizeof first published struct */ /* @@ -102,6 +104,10 @@ void su_exit(void); * Given this task model, there are a multiplicity of scheduling algorithms * and policies, that can be used to ensure all the tasks will make their * timing constraints. + * + * As of now, the SCHED_DEADLINE policy (sched_dl scheduling class) is the + * only user of this new interface. More information about the algorithm + * available in the scheduling class file or in Documentation/. */ struct sched_attr { u32 size; @@ -128,12 +134,11 @@ struct bio_list; struct fs_struct; struct perf_event_context; struct blk_plug; +struct filename; -/* - * List of flags we want to share for kernel threads, - * if only because they are not used by them anyway. - */ -#define CLONE_KERNEL (CLONE_FS | CLONE_FILES | CLONE_SIGHAND) +#define VMACACHE_BITS 2 +#define VMACACHE_SIZE (1U << VMACACHE_BITS) +#define VMACACHE_MASK (VMACACHE_SIZE - 1) /* * These are the constant used to fake the fixed-point load-average @@ -165,23 +170,21 @@ extern int nr_threads; DECLARE_PER_CPU(unsigned long, process_counts); extern int nr_processes(void); extern unsigned long nr_running(void); +extern bool single_task_running(void); extern unsigned long nr_iowait(void); extern unsigned long nr_iowait_cpu(int cpu); -extern unsigned long this_cpu_load(void); +extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load); extern void sched_update_nr_prod(int cpu, long delta, bool inc); extern void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg); extern void calc_global_load(unsigned long ticks); -extern void update_cpu_load_nohz(void); -/* Notifier for when a task gets migrated to a new CPU */ -struct task_migration_notifier { - struct task_struct *task; - int from_cpu; - int to_cpu; -}; -extern void register_task_migration_notifier(struct notifier_block *n); +#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) +extern void update_cpu_load_nohz(void); +#else +static inline void update_cpu_load_nohz(void) { } +#endif extern unsigned long get_parent_ip(unsigned long addr); @@ -213,8 +216,9 @@ print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq); #define __TASK_STOPPED 4 #define __TASK_TRACED 8 /* in tsk->exit_state */ -#define EXIT_ZOMBIE 16 -#define EXIT_DEAD 32 +#define EXIT_DEAD 16 +#define EXIT_ZOMBIE 32 +#define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD) /* in tsk->state again */ #define TASK_DEAD 64 #define TASK_WAKEKILL 128 @@ -222,7 +226,7 @@ print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq); #define TASK_PARKED 512 #define TASK_STATE_MAX 1024 -#define TASK_STATE_TO_CHAR_STR "RSDTtZXxKWP" +#define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWP" extern char ___assert_task_state[1 - 2*!!( sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1)]; @@ -239,11 +243,10 @@ extern char ___assert_task_state[1 - 2*!!( /* get_task_state() */ #define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \ TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \ - __TASK_TRACED) + __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD) #define task_is_traced(task) ((task->state & __TASK_TRACED) != 0) #define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0) -#define task_is_dead(task) ((task)->exit_state != 0) #define task_is_stopped_or_traced(task) \ ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0) #define task_contributes_to_load(task) \ @@ -317,6 +320,10 @@ extern int get_nohz_timer_target(void); #else static inline void nohz_balance_enter_idle(int cpu) { } static inline void set_cpu_sd_state_idle(void) { } +static inline int get_nohz_timer_target(void) +{ + return smp_processor_id(); +} #endif /* @@ -372,6 +379,14 @@ static inline void lockup_detector_init(void) } #endif +#ifdef CONFIG_DETECT_HUNG_TASK +void reset_hung_task_detector(void); +#else +static inline void reset_hung_task_detector(void) +{ +} +#endif + /* Attach to any functions which should be ignored in wchan output. */ #define __sched __attribute__((__section__(".sched.text"))) @@ -405,22 +420,38 @@ arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, static inline void arch_pick_mmap_layout(struct mm_struct *mm) {} #endif - -extern void set_dumpable(struct mm_struct *mm, int value); -extern int get_dumpable(struct mm_struct *mm); - #define SUID_DUMP_DISABLE 0 /* No setuid dumping */ #define SUID_DUMP_USER 1 /* Dump as user of process */ #define SUID_DUMP_ROOT 2 /* Dump as root */ /* mm flags */ -/* dumpable bits */ -#define MMF_DUMPABLE 0 /* core dump is permitted */ -#define MMF_DUMP_SECURELY 1 /* core file is readable only by root */ +/* for SUID_DUMP_* above */ +#define MMF_DUMPABLE 0 +#define MMF_DUMP_SECURELY 1 #define MMF_DUMPABLE_BITS 2 #define MMF_DUMPABLE_MASK ((1 << MMF_DUMPABLE_BITS) - 1) +extern void set_dumpable(struct mm_struct *mm, int value); +extern int get_dumpable(struct mm_struct *mm); +/* + * This returns the actual value of the suid_dumpable flag. For things + * that are using this for checking for privilege transitions, it must + * test against SUID_DUMP_USER rather than treating it as a boolean + * value. + */ +#if 0 +static inline int __get_dumpable(unsigned long mm_flags) +{ + return mm_flags & MMF_DUMPABLE_MASK; +} + +static inline int get_dumpable(struct mm_struct *mm) +{ + return __get_dumpable(mm->flags); +} +#endif + /* coredump filter bits */ #define MMF_DUMP_ANON_PRIVATE 2 #define MMF_DUMP_ANON_SHARED 3 @@ -518,6 +549,14 @@ struct task_cputime { .sum_exec_runtime = 0, \ } +#ifdef TJK_HMP +#ifdef CONFIG_PREEMPT_COUNT +#define PREEMPT_DISABLED (1 + PREEMPT_ENABLED) +#else +#define PREEMPT_DISABLED PREEMPT_ENABLED +#endif +#endif + /* * Disable preemption until the scheduler is running. * Reset by start_kernel()->sched_init()->init_idle(). @@ -636,6 +675,7 @@ struct signal_struct { * Live threads maintain their own counters and add to these * in __exit_signal, except for the group leader. */ + seqlock_t stats_lock; cputime_t utime, stime, cutime, cstime; cputime_t gtime; cputime_t cgtime; @@ -730,7 +770,7 @@ static inline int signal_group_exit(const struct signal_struct *sig) struct user_struct { atomic_t __count; /* reference count */ atomic_t processes; /* How many processes does this user have? */ - atomic_t files; /* How many open files does this user have? */ + atomic_t files; atomic_t sigpending; /* How many pending signals does this user have? */ #ifdef CONFIG_INOTIFY_USER atomic_t inotify_watches; /* How many inotify watches does this user have? */ @@ -807,7 +847,7 @@ struct task_delay_info { * associated with the operation is added to XXX_delay. * XXX_delay contains the accumulated delay time in nanoseconds. */ - struct timespec blkio_start, blkio_end; /* Shared by blkio, swapin */ + u64 blkio_start; /* Shared by blkio, swapin */ u64 blkio_delay; /* wait for sync block io completion */ u64 swapin_delay; /* wait for swapin block io completion */ u32 blkio_count; /* total count of the number of sync block */ @@ -815,7 +855,7 @@ struct task_delay_info { u32 swapin_count; /* total count of the number of swapin block */ /* io operations performed */ - struct timespec freepages_start, freepages_end; + u64 freepages_start; u64 freepages_delay; /* wait for memory reclaim */ u32 freepages_count; /* total count of memory reclaim */ }; @@ -841,10 +881,10 @@ enum cpu_idle_type { }; /* - * Increase resolution of cpu_power calculations + * Increase resolution of cpu_capacity calculations */ -#define SCHED_POWER_SHIFT 10 -#define SCHED_POWER_SCALE (1L << SCHED_POWER_SHIFT) +#define SCHED_CAPACITY_SHIFT 10 +#define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT) /* * sched-domains (multiprocessor balancing) declarations: @@ -856,14 +896,36 @@ enum cpu_idle_type { #define SD_BALANCE_FORK 0x0008 /* Balance on fork, clone */ #define SD_BALANCE_WAKE 0x0010 /* Balance on wakeup */ #define SD_WAKE_AFFINE 0x0020 /* Wake task to waking CPU */ -#define SD_SHARE_CPUPOWER 0x0080 /* Domain members share cpu power */ +#define SD_SHARE_CPUCAPACITY 0x0080 /* Domain members share cpu power */ +#define SD_SHARE_POWERDOMAIN 0x0100 /* Domain members share power domain */ #define SD_SHARE_PKG_RESOURCES 0x0200 /* Domain members share cpu pkg resources */ #define SD_SERIALIZE 0x0400 /* Only a single load balancing instance */ #define SD_ASYM_PACKING 0x0800 /* Place busy groups earlier in the domain */ #define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */ #define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */ +#define SD_NUMA 0x4000 /* cross-node balancing */ +#define SD_SHARE_CAP_STATES 0x8000 /* Domain members share capacity state */ + +#ifdef CONFIG_SCHED_SMT +static inline int cpu_smt_flags(void) +{ + return SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES; +} +#endif + +#ifdef CONFIG_SCHED_MC +static inline int cpu_core_flags(void) +{ + return SD_SHARE_PKG_RESOURCES; +} +#endif -extern int __weak arch_sd_sibiling_asym_packing(void); +#ifdef CONFIG_NUMA +static inline int cpu_numa_flags(void) +{ + return SD_NUMA; +} +#endif struct sched_domain_attr { int relax_domain_level; @@ -875,6 +937,26 @@ struct sched_domain_attr { extern int sched_domain_level_max; +struct capacity_state { + unsigned long cap; /* compute capacity */ + unsigned long power; /* power consumption at this compute capacity */ +}; + +struct idle_state { + unsigned long power; /* power consumption in this idle state */ +}; + +struct sched_group_energy { + atomic_t ref; + unsigned int nr_idle_states; /* number of idle states */ + struct idle_state *idle_states; /* ptr to idle state array */ + unsigned int nr_idle_states_below; /* number idle states in lower groups */ + unsigned int nr_cap_states; /* number of capacity states */ + struct capacity_state *cap_states; /* ptr to capacity state array */ +}; + +unsigned long capacity_curr_of(int cpu); + struct sched_group; struct sched_domain { @@ -903,7 +985,9 @@ struct sched_domain { unsigned int balance_interval; /* initialise to 1. units in ms. */ unsigned int nr_balance_failed; /* initialise to 0 */ - u64 last_update; + /* idle_balance() stats */ + u64 max_newidle_lb_cost; + unsigned long next_decay_max_lb_cost; #ifdef CONFIG_SCHEDSTATS /* load_balance() stats */ @@ -969,6 +1053,42 @@ void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms); bool cpus_share_cache(int this_cpu, int that_cpu); +typedef const struct cpumask *(*sched_domain_mask_f)(int cpu); +typedef int (*sched_domain_flags_f)(void); +typedef const struct sched_group_energy *(*sched_domain_energy_f)(int cpu); + +#define SDTL_OVERLAP 0x01 + +struct sd_data { + struct sched_domain **__percpu sd; + struct sched_group **__percpu sg; + struct sched_group_capacity **__percpu sgc; + struct sched_group_energy **__percpu sge; +}; + +struct sched_domain_topology_level { + sched_domain_mask_f mask; + sched_domain_flags_f sd_flags; + sched_domain_energy_f energy; + int flags; + int numa_level; + struct sd_data data; +#ifdef CONFIG_SCHED_DEBUG + char *name; +#endif +}; + +extern struct sched_domain_topology_level *sched_domain_topology; + +extern void set_sched_topology(struct sched_domain_topology_level *tl); +extern void wake_up_if_idle(int cpu); + +#ifdef CONFIG_SCHED_DEBUG +# define SD_INIT_NAME(type) .name = #type +#else +# define SD_INIT_NAME(type) +#endif + #else /* CONFIG_SMP */ struct sched_domain_attr; @@ -1002,24 +1122,35 @@ struct pipe_inode_info; struct uts_namespace; struct load_weight { - unsigned long weight, inv_weight; + unsigned long weight; + u32 inv_weight; }; struct sched_avg { - /* - * These sums represent an infinite geometric series and so are bound - * above by 1024/(1-y). Thus we only need a u32 to store them for for all - * choices of y < 1-2^(-32)*1024. - */ - u32 runnable_avg_sum, runnable_avg_period; -#ifdef CONFIG_SCHED_HMP - u32 runnable_avg_sum_scaled; -#endif u64 last_runnable_update; s64 decay_count; - unsigned long load_avg_contrib; + /* + * utilization_avg_contrib describes the amount of time that a + * sched_entity is running on a CPU. It is based on running_avg_sum + * and is scaled in the range [0..SCHED_LOAD_SCALE]. + * load_avg_contrib described the amount of time that a sched_entity + * is runnable on a rq. It is based on both runnable_avg_sum and the + * weight of the task. + */ + unsigned long load_avg_contrib, utilization_avg_contrib; + /* + * These sums represent an infinite geometric series and so are bound + * above by 1024/(1-y). Thus we only need a u32 to store them for all + * choices of y < 1-2^(-32)*1024. + * running_avg_sum reflects the time that the sched_entity is + * effectively running on the CPU. + * runnable_avg_sum represents the amount of time a sched_entity is on + * a runqueue which includes the running time that is monitored by + * running_avg_sum. + */ + u32 runnable_avg_sum, avg_period, running_avg_sum; }; - + #ifdef CONFIG_SCHEDSTATS struct sched_statistics { u64 wait_start; @@ -1085,11 +1216,9 @@ struct ravg { u64 mark_start; u32 sum, demand; u32 sum_history[RAVG_HIST_SIZE_MAX]; -#ifdef VENDOR_EDIT - unsigned mitigated:1; -#endif #ifdef CONFIG_SCHED_FREQ_INPUT u32 curr_window, prev_window; + u16 active_windows; #endif }; @@ -1111,6 +1240,7 @@ struct sched_entity { #endif #ifdef CONFIG_FAIR_GROUP_SCHED + int depth; struct sched_entity *parent; /* rq on which this entity is (to be) queued: */ struct cfs_rq *cfs_rq; @@ -1118,12 +1248,7 @@ struct sched_entity { struct cfs_rq *my_q; #endif -/* - * Load-tracking only depends on SMP, FAIR_GROUP_SCHED dependency below may be - * removed when useful for applications beyond shares distribution (e.g. - * load-balance). - */ -#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED) +#ifdef CONFIG_SMP /* Per-entity load-tracking */ struct sched_avg avg; #endif @@ -1145,7 +1270,62 @@ struct sched_rt_entity { #endif }; +struct sched_dl_entity { + struct rb_node rb_node; + + /* + * Original scheduling parameters. Copied here from sched_attr + * during sched_setattr(), they will remain the same until + * the next sched_setattr(). + */ + u64 dl_runtime; /* maximum runtime for each instance */ + u64 dl_deadline; /* relative deadline of each instance */ + u64 dl_period; /* separation of two instances (period) */ + u64 dl_bw; /* dl_runtime / dl_deadline */ + + /* + * Actual scheduling parameters. Initialized with the values above, + * they are continously updated during task execution. Note that + * the remaining runtime could be < 0 in case we are in overrun. + */ + s64 runtime; /* remaining runtime for this instance */ + u64 deadline; /* absolute deadline for this instance */ + unsigned int flags; /* specifying the scheduler behaviour */ + + /* + * Some bool flags: + * + * @dl_throttled tells if we exhausted the runtime. If so, the + * task has to wait for a replenishment to be performed at the + * next firing of dl_timer. + * + * @dl_new tells if a new instance arrived. If so we must + * start executing it with full runtime and reset its absolute + * deadline; + * + * @dl_boosted tells if we are boosted due to DI. If so we are + * outside bandwidth enforcement mechanism (but only until we + * exit the critical section); + * + * @dl_yielded tells if task gave up the cpu before consuming + * all its available runtime during the last job. + */ + int dl_throttled, dl_new, dl_boosted, dl_yielded; + + /* + * Bandwidth enforcement timer. Each -deadline task has its + * own bandwidth to be enforced, thus we need one timer per task. + */ + struct hrtimer dl_timer; +}; +union rcu_special { + struct { + bool blocked; + bool need_qs; + } b; + short s; +}; struct rcu_node; enum perf_event_task_context { @@ -1161,14 +1341,20 @@ struct task_struct { atomic_t usage; unsigned int flags; /* per process flags, defined below */ unsigned int ptrace; - #ifdef VENDOR_EDIT - //huruihuan add for kill task in D status + #ifdef VENDOR_EDIT + //huruihuan add for kill task in D status unsigned int kill_flag; struct timespec ttu; - #endif + #endif + #ifdef CONFIG_SMP struct llist_node wake_entry; int on_cpu; + struct task_struct *last_wakee; + unsigned long wakee_flips; + unsigned long wakee_flip_decay_ts; + + int wake_cpu; #endif int on_rq; @@ -1184,26 +1370,21 @@ struct task_struct { * of this task */ u32 init_load_pct; + u64 last_wake_ts; +#ifdef CONFIG_SCHED_QHMP u64 run_start; #endif +#endif #ifdef CONFIG_CGROUP_SCHED struct task_group *sched_task_group; #endif + struct sched_dl_entity dl; #ifdef CONFIG_PREEMPT_NOTIFIERS /* list of struct preempt_notifier: */ struct hlist_head preempt_notifiers; #endif - /* - * fpu_counter contains the number of consecutive context switches - * that the FPU is used. If this is over a threshold, the lazy fpu - * saving becomes unlazy to save the trap. This is an unsigned char - * so that after 256 times the counter wraps and the behavior turns - * lazy again; this to deal with bursty apps that only use FPU for - * a short time - */ - unsigned char fpu_counter; #ifdef CONFIG_BLK_DEV_IO_TRACE unsigned int btrace_seq; #endif @@ -1220,9 +1401,12 @@ struct task_struct { #ifdef CONFIG_TREE_PREEMPT_RCU struct rcu_node *rcu_blocked_node; #endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */ -#ifdef CONFIG_RCU_BOOST - struct rt_mutex *rcu_boost_mutex; -#endif /* #ifdef CONFIG_RCU_BOOST */ +#ifdef CONFIG_TASKS_RCU + unsigned long rcu_tasks_nvcsw; + bool rcu_tasks_holdout; + struct list_head rcu_tasks_holdout_list; + int rcu_tasks_idle_cpu; +#endif /* #ifdef CONFIG_TASKS_RCU */ #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) struct sched_info sched_info; @@ -1231,12 +1415,16 @@ struct task_struct { struct list_head tasks; #ifdef CONFIG_SMP struct plist_node pushable_tasks; + struct rb_node pushable_dl_tasks; #endif struct mm_struct *mm, *active_mm; #ifdef CONFIG_COMPAT_BRK unsigned brk_randomized:1; #endif + /* per-thread vma caching */ + u32 vmacache_seqnum; + struct vm_area_struct *vmacache[VMACACHE_SIZE]; #if defined(SPLIT_RSS_COUNTING) struct task_rss_stat rss_stat; #endif @@ -1314,8 +1502,8 @@ struct task_struct { } vtime_snap_whence; #endif unsigned long nvcsw, nivcsw; /* context switch counts */ - struct timespec start_time; /* monotonic time */ - struct timespec real_start_time; /* boot based time */ + struct timespec start_time; /* monotonic time in nsec */ + struct timespec real_start_time; /* boot based time in nsec */ /* mm fault and swap info: this can arguably be seen as either mm-specific or thread-specific */ unsigned long min_flt, maj_flt; @@ -1336,6 +1524,7 @@ struct task_struct { #ifdef CONFIG_SYSVIPC /* ipc stuff */ struct sysv_sem sysvsem; + struct sysv_shm sysvshm; #endif #ifdef CONFIG_DETECT_HUNG_TASK /* hung task detection */ @@ -1467,6 +1656,9 @@ struct task_struct { struct mutex perf_event_mutex; struct list_head perf_event_list; #endif +#ifdef CONFIG_DEBUG_PREEMPT + unsigned long preempt_disable_ip; +#endif #ifdef CONFIG_NUMA struct mempolicy *mempolicy; /* Protected by alloc_lock */ short il_next; @@ -1474,10 +1666,49 @@ struct task_struct { #endif #ifdef CONFIG_NUMA_BALANCING int numa_scan_seq; - int numa_migrate_seq; unsigned int numa_scan_period; + unsigned int numa_scan_period_max; + int numa_preferred_nid; + unsigned long numa_migrate_retry; u64 node_stamp; /* migration stamp */ + u64 last_task_numa_placement; + u64 last_sum_exec_runtime; struct callback_head numa_work; + + struct list_head numa_entry; + struct numa_group *numa_group; + + /* + * Exponential decaying average of faults on a per-node basis. + * Scheduling placement decisions are made based on the these counts. + * The values remain static for the duration of a PTE scan + */ + unsigned long *numa_faults_memory; + unsigned long total_numa_faults; + + /* + * numa_faults_buffer records faults per node during the current + * scan window. When the scan completes, the counts in + * numa_faults_memory decay and these values are copied. + */ + unsigned long *numa_faults_buffer_memory; + + /* + * Track the nodes the process was running on when a NUMA hinting + * fault was incurred. + */ + unsigned long *numa_faults_cpu; + unsigned long *numa_faults_buffer_cpu; + + /* + * numa_faults_locality tracks if faults recorded during the last + * scan window were remote/local. The task scan period is adapted + * based on the locality of the faults with different weights + * depending on whether they were shared or private faults + */ + unsigned long numa_faults_locality[2]; + + unsigned long numa_pages_migrated; #endif /* CONFIG_NUMA_BALANCING */ struct rcu_head rcu; @@ -1514,6 +1745,9 @@ struct task_struct { unsigned long timer_slack_ns; unsigned long default_timer_slack_ns; +#ifdef CONFIG_KASAN + unsigned int kasan_depth; +#endif #ifdef CONFIG_FUNCTION_GRAPH_TRACER /* Index of current stored address in ret_stack */ int curr_ret_stack; @@ -1536,12 +1770,6 @@ struct task_struct { unsigned long trace_recursion; #endif /* CONFIG_TRACING */ #ifdef CONFIG_MEMCG /* memcg uses this to do batch job */ - struct memcg_batch_info { - int do_batch; /* incremented when batch uncharge started */ - struct mem_cgroup *memcg; /* target memcg of uncharge */ - unsigned long nr_pages; /* uncharged usage */ - unsigned long memsw_nr_pages; /* uncharged mem+swap usage */ - } memcg_batch; unsigned int memcg_kmem_skip_account; struct memcg_oom_info { struct mem_cgroup *memcg; @@ -1550,9 +1778,6 @@ struct task_struct { unsigned int may_oom:1; } memcg_oom; #endif -#ifdef CONFIG_HAVE_HW_BREAKPOINT - atomic_t ptrace_bp_refcnt; -#endif #ifdef CONFIG_UPROBES struct uprobe_task *utask; #endif @@ -1565,16 +1790,37 @@ struct task_struct { /* Future-safe accessor for struct task_struct's cpus_allowed. */ #define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) +#define TNF_MIGRATED 0x01 +#define TNF_NO_GROUP 0x02 +#define TNF_SHARED 0x04 +#define TNF_FAULT_LOCAL 0x08 + #ifdef CONFIG_NUMA_BALANCING -extern void task_numa_fault(int node, int pages, bool migrated); +extern void task_numa_fault(int node, int pages, int migrated); +extern pid_t task_numa_group_id(struct task_struct *p); extern void set_numabalancing_state(bool enabled); +extern void task_numa_free(struct task_struct *p); +extern bool should_numa_migrate_memory(struct task_struct *p, struct page *page, + int src_nid, int dst_cpu); #else -static inline void task_numa_fault(int node, int pages, bool migrated) +static inline void task_numa_fault(int node, int pages, int migrated) +{ +} +static inline pid_t task_numa_group_id(struct task_struct *p) { + return 0; } static inline void set_numabalancing_state(bool enabled) { } +static inline void task_numa_free(struct task_struct *p) +{ +} +static inline bool should_numa_migrate_memory(struct task_struct *p, + struct page *page, int src_nid, int dst_cpu) +{ + return true; +} #endif static inline struct pid *task_pid(struct task_struct *task) @@ -1650,6 +1896,24 @@ static inline pid_t task_tgid_vnr(struct task_struct *tsk) } +static inline int pid_alive(const struct task_struct *p); +static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns) +{ + pid_t pid = 0; + + rcu_read_lock(); + if (pid_alive(tsk)) + pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns); + rcu_read_unlock(); + + return pid; +} + +static inline pid_t task_ppid_nr(const struct task_struct *tsk) +{ + return task_ppid_nr_ns(tsk, &init_pid_ns); +} + static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns) { @@ -1686,8 +1950,10 @@ static inline pid_t task_pgrp_nr(struct task_struct *tsk) * Test if a process is not yet dead (at most zombie state) * If pid_alive fails, then pointers within the task structure * can be stale and must not be dereferenced. + * + * Return: 1 if the process is alive. 0 otherwise. */ -static inline int pid_alive(struct task_struct *p) +static inline int pid_alive(const struct task_struct *p) { return p->pids[PIDTYPE_PID].pid != NULL; } @@ -1697,6 +1963,8 @@ static inline int pid_alive(struct task_struct *p) * @tsk: Task structure to be checked. * * Check if a task structure is the first user space task the kernel created. + * + * Return: 1 if the task structure is init. 0 otherwise. */ static inline int is_global_init(struct task_struct *tsk) { @@ -1752,12 +2020,26 @@ extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, extern int task_free_register(struct notifier_block *n); extern int task_free_unregister(struct notifier_block *n); -#ifdef CONFIG_SCHED_FREQ_INPUT + +struct sched_load { + unsigned long prev_load; + unsigned long new_task_load; +}; + +#if defined(CONFIG_SCHED_FREQ_INPUT) extern int sched_set_window(u64 window_start, unsigned int window_size); extern unsigned long sched_get_busy(int cpu); -extern void sched_get_cpus_busy(unsigned long *busy, +extern void sched_get_cpus_busy(struct sched_load *busy, const struct cpumask *query_cpus); extern void sched_set_io_is_busy(int val); +#ifdef CONFIG_SCHED_QHMP +static inline int sched_update_freq_max_load(const cpumask_t *cpumask) +{ + return 0; +} +#else +int sched_update_freq_max_load(const cpumask_t *cpumask); +#endif #else static inline int sched_set_window(u64 window_start, unsigned int window_size) { @@ -1767,14 +2049,20 @@ static inline unsigned long sched_get_busy(int cpu) { return 0; } -static inline void sched_get_cpus_busy(unsigned long *busy, - const struct cpumask *query_cpus) {}; +static inline void sched_get_cpus_busy(struct sched_load *busy, + const struct cpumask *query_cpus) {}; static inline void sched_set_io_is_busy(int val) {}; + +static inline int sched_update_freq_max_load(const cpumask_t *cpumask) +{ + return 0; +} #endif /* * Per process flags */ +#define PF_WAKE_UP_IDLE 0x00000002 /* try to wake up on an idle CPU */ #define PF_EXITING 0x00000004 /* getting shut down */ #define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */ #define PF_VCPU 0x00000010 /* I'm a virtual CPU */ @@ -1797,16 +2085,13 @@ static inline void sched_set_io_is_busy(int val) {}; #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ #define PF_RANDOMIZE 0x00400000 /* randomize virtual address space */ #define PF_SWAPWRITE 0x00800000 /* Allowed to write to swap */ -#define PF_SPREAD_PAGE 0x01000000 /* Spread page cache over cpuset */ -#define PF_SPREAD_SLAB 0x02000000 /* Spread some slab caches over cpuset */ #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_allowed */ #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */ -#define PF_MEMPOLICY 0x10000000 /* Non-default NUMA mempolicy */ +#define PF_SPREAD_PAGE 0x01000000 +#define PF_SPREAD_SLAB 0x02000000 #define PF_MUTEX_TESTER 0x20000000 /* Thread belongs to the rt mutex tester */ #define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */ -#define PF_WAKE_UP_IDLE 0x80000000 /* try to wake up on an idle CPU */ - -#define PF_SU 0x00000002 /* task is su */ +#define PF_SUSPEND_TASK 0x80000000 /* this thread called freeze_processes and should not be frozen */ /* * Only the _current_ task can read/write to tsk->flags, but other @@ -1856,17 +2141,31 @@ static inline void memalloc_noio_restore(unsigned int flags) } /* Per-process atomic flags. */ -#define PFA_NO_NEW_PRIVS 0x00000001 /* May not gain new privileges. */ +#define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */ +#define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */ +#define PFA_SPREAD_SLAB 2 /* Spread some slab caches over cpuset */ -static inline bool task_no_new_privs(struct task_struct *p) -{ - return test_bit(PFA_NO_NEW_PRIVS, &p->atomic_flags); -} -static inline void task_set_no_new_privs(struct task_struct *p) -{ - set_bit(PFA_NO_NEW_PRIVS, &p->atomic_flags); -} +#define TASK_PFA_TEST(name, func) \ + static inline bool task_##func(struct task_struct *p) \ + { return test_bit(PFA_##name, &p->atomic_flags); } +#define TASK_PFA_SET(name, func) \ + static inline void task_set_##func(struct task_struct *p) \ + { set_bit(PFA_##name, &p->atomic_flags); } +#define TASK_PFA_CLEAR(name, func) \ + static inline void task_clear_##func(struct task_struct *p) \ + { clear_bit(PFA_##name, &p->atomic_flags); } + +TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs) +TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs) + +TASK_PFA_TEST(SPREAD_PAGE, spread_page) +TASK_PFA_SET(SPREAD_PAGE, spread_page) +TASK_PFA_CLEAR(SPREAD_PAGE, spread_page) + +TASK_PFA_TEST(SPREAD_SLAB, spread_slab) +TASK_PFA_SET(SPREAD_SLAB, spread_slab) +TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab) /* * task->jobctl flags @@ -1897,33 +2196,24 @@ extern bool task_set_jobctl_pending(struct task_struct *task, extern void task_clear_jobctl_trapping(struct task_struct *task); extern void task_clear_jobctl_pending(struct task_struct *task, unsigned int mask); - -#ifdef CONFIG_PREEMPT_RCU - #define RCU_READ_UNLOCK_BLOCKED (1 << 0) /* blocked while in RCU read-side. */ #define RCU_READ_UNLOCK_NEED_QS (1 << 1) /* RCU core needs CPU response. */ static inline void rcu_copy_process(struct task_struct *p) { +#ifdef CONFIG_PREEMPT_RCU p->rcu_read_lock_nesting = 0; p->rcu_read_unlock_special = 0; -#ifdef CONFIG_TREE_PREEMPT_RCU p->rcu_blocked_node = NULL; -#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */ -#ifdef CONFIG_RCU_BOOST - p->rcu_boost_mutex = NULL; -#endif /* #ifdef CONFIG_RCU_BOOST */ INIT_LIST_HEAD(&p->rcu_node_entry); +#endif /* #ifdef CONFIG_PREEMPT_RCU */ +#ifdef CONFIG_TASKS_RCU + p->rcu_tasks_holdout = false; + INIT_LIST_HEAD(&p->rcu_tasks_holdout_list); + p->rcu_tasks_idle_cpu = -1; +#endif /* #ifdef CONFIG_TASKS_RCU */ } -#else - -static inline void rcu_copy_process(struct task_struct *p) -{ -} - -#endif - static inline void tsk_restore_flags(struct task_struct *task, unsigned long orig_flags, unsigned long flags) { @@ -1932,13 +2222,17 @@ static inline void tsk_restore_flags(struct task_struct *task, } #ifdef CONFIG_SMP +struct cpuidle_state; extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask); extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask); extern void sched_set_cpu_cstate(int cpu, int cstate, + struct cpuidle_state *cpuidle, int wakeup_energy, int wakeup_latency); +extern void sched_set_cluster_dstate(const cpumask_t *cluster_cpus, int dstate, + int wakeup_energy, int wakeup_latency); #else static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) @@ -1952,7 +2246,13 @@ static inline int set_cpus_allowed_ptr(struct task_struct *p, return 0; } static inline void -sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency) +sched_set_cpu_cstate(int cpu, int cstate, struct cpuidle *cpuidle, + int wakeup_energy, int wakeup_latency) +{ +} + +static inline void sched_set_cluster_dstate(const cpumask_t *cluster_cpus, + int dstate, int wakeup_energy, int wakeup_latency) { } #endif @@ -1965,6 +2265,11 @@ extern u32 sched_get_wake_up_idle(struct task_struct *p); extern int sched_set_boost(int enable); extern int sched_set_init_task_load(struct task_struct *p, int init_load_pct); extern u32 sched_get_init_task_load(struct task_struct *p); +extern int sched_set_static_cpu_pwr_cost(int cpu, unsigned int cost); +extern unsigned int sched_get_static_cpu_pwr_cost(int cpu); +extern int sched_set_static_cluster_pwr_cost(int cpu, unsigned int cost); +extern unsigned int sched_get_static_cluster_pwr_cost(int cpu); +#ifdef CONFIG_SCHED_QHMP extern int sched_set_cpu_prefer_idle(int cpu, int prefer_idle); extern int sched_get_cpu_prefer_idle(int cpu); extern int sched_set_cpu_mostly_idle_load(int cpu, int mostly_idle_pct); @@ -1974,6 +2279,7 @@ extern int sched_get_cpu_mostly_idle_nr_run(int cpu); extern int sched_set_cpu_mostly_idle_freq(int cpu, unsigned int mostly_idle_freq); extern unsigned int sched_get_cpu_mostly_idle_freq(int cpu); +#endif #else static inline int sched_set_boost(int enable) @@ -2044,7 +2350,9 @@ static inline void sched_clock_idle_wakeup_event(u64 delta_ns) * but then during bootup it turns out that sched_clock() * is reliable after all: */ -extern int sched_clock_stable; +extern int sched_clock_stable(void); +extern void set_sched_clock_stable(void); +extern void clear_sched_clock_stable(void); extern void sched_clock_tick(void); extern void sched_clock_idle_sleep_event(void); @@ -2112,10 +2420,19 @@ static inline void sched_autogroup_fork(struct signal_struct *sig) { } static inline void sched_autogroup_exit(struct signal_struct *sig) { } #endif -extern bool yield_to(struct task_struct *p, bool preempt); +extern int yield_to(struct task_struct *p, bool preempt); extern void set_user_nice(struct task_struct *p, long nice); extern int task_prio(const struct task_struct *p); -extern int task_nice(const struct task_struct *p); +/** + * task_nice - return the nice value of a given task. + * @p: the task in question. + * + * Return: The nice value [ -20 ... 0 ... 19 ]. + */ +static inline int task_nice(const struct task_struct *p) +{ + return PRIO_TO_NICE((p)->static_prio); +} extern int can_nice(const struct task_struct *p, const int nice); extern int task_curr(const struct task_struct *p); extern int idle_cpu(int cpu); @@ -2129,6 +2446,8 @@ extern struct task_struct *idle_task(int cpu); /** * is_idle_task - is the specified task an idle task? * @p: the task in question. + * + * Return: 1 if @p is an idle task. 0 otherwise. */ static inline bool is_idle_task(const struct task_struct *p) { @@ -2182,7 +2501,6 @@ extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns); extern void __set_special_pids(struct pid *pid); - /* per-UID process charging. */ extern struct user_struct * alloc_uid(kuid_t); static inline struct user_struct *get_uid(struct user_struct *u) @@ -2204,7 +2522,7 @@ extern void wake_up_new_task(struct task_struct *tsk); #else static inline void kick_process(struct task_struct *tsk) { } #endif -extern void sched_fork(struct task_struct *p); +extern int sched_fork(unsigned long clone_flags, struct task_struct *p); extern void sched_dead(struct task_struct *p); #ifdef CONFIG_SCHED_HMP extern void sched_exit(struct task_struct *p); @@ -2295,8 +2613,10 @@ static inline int on_sig_stack(unsigned long sp) static inline int sas_ss_flags(unsigned long sp) { - return (current->sas_ss_size == 0 ? SS_DISABLE - : on_sig_stack(sp) ? SS_ONSTACK : 0); + if (!current->sas_ss_size) + return SS_DISABLE; + + return on_sig_stack(sp) ? SS_ONSTACK : 0; } static inline unsigned long sigsp(unsigned long sp, struct ksignal *ksig) @@ -2335,8 +2655,6 @@ extern struct mm_struct *get_task_mm(struct task_struct *task); extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode); /* Remove the current tasks stale references to the old mm_struct */ extern void mm_release(struct task_struct *, struct mm_struct *); -/* Allocate a new mm structure and copy contents from tsk->mm */ -extern struct mm_struct *dup_mm(struct task_struct *tsk); extern int copy_thread(unsigned long, unsigned long, unsigned long, struct task_struct *); @@ -2361,6 +2679,7 @@ extern long do_fork(unsigned long, unsigned long, unsigned long, int __user *, i struct task_struct *fork_idle(int); extern pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags); +extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec); extern void set_task_comm(struct task_struct *tsk, char *from); extern char *get_task_comm(char *to, struct task_struct *tsk); @@ -2543,12 +2862,27 @@ static inline void setup_thread_stack(struct task_struct *p, struct task_struct task_thread_info(p)->task = p; } +/* + * Return the address of the last usable long on the stack. + * + * When the stack grows down, this is just above the thread + * info struct. Going any lower will corrupt the threadinfo. + * + * When the stack grows up, this is the highest address. + * Beyond that position, we corrupt data on the next page. + */ static inline unsigned long *end_of_stack(struct task_struct *p) { +#ifdef CONFIG_STACK_GROWSUP + return (unsigned long *)((unsigned long)task_thread_info(p) + THREAD_SIZE) - 1; +#else return (unsigned long *)(task_thread_info(p) + 1); +#endif } #endif +#define task_stack_end_corrupted(task) \ + (*(end_of_stack(task)) != STACK_END_MAGIC) static inline int object_is_on_stack(void *obj) { @@ -2571,6 +2905,7 @@ static inline unsigned long stack_not_used(struct task_struct *p) return (unsigned long)n - (unsigned long)end_of_stack(p); } #endif +extern void set_task_stack_end_magic(struct task_struct *tsk); /* set thread flags in other task's structures * - see asm/thread_info.h for TIF_xxxx flags available @@ -2646,11 +2981,6 @@ static inline int signal_pending_state(long state, struct task_struct *p) return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p); } -static inline int need_resched(void) -{ - return unlikely(test_thread_flag(TIF_NEED_RESCHED)); -} - /* * cond_resched() and cond_resched_lock(): latency reduction via * explicit rescheduling in places that are safe. The return @@ -2685,6 +3015,15 @@ extern int __cond_resched_softirq(void); __cond_resched_softirq(); \ }) +static inline void cond_resched_rcu(void) +{ +#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU) + rcu_read_unlock(); + cond_resched(); + rcu_read_lock(); +#endif +} + /* * Does a critical section need to be broken due to another * task waiting?: (technically does not depend on CONFIG_PREEMPT, @@ -2701,51 +3040,9 @@ static inline int spin_needbreak(spinlock_t *lock) /* * Idle thread specific functions to determine the need_resched - * polling state. We have two versions, one based on TS_POLLING in - * thread_info.status and one based on TIF_POLLING_NRFLAG in - * thread_info.flags + * polling state. */ -#ifdef TS_POLLING -static inline int tsk_is_polling(struct task_struct *p) -{ - return task_thread_info(p)->status & TS_POLLING; -} -static inline void __current_set_polling(void) -{ - current_thread_info()->status |= TS_POLLING; -} - -static inline bool __must_check current_set_polling_and_test(void) -{ - __current_set_polling(); - - /* - * Polling state must be visible before we test NEED_RESCHED, - * paired by resched_task() - */ - smp_mb(); - - return unlikely(tif_need_resched()); -} - -static inline void __current_clr_polling(void) -{ - current_thread_info()->status &= ~TS_POLLING; -} - -static inline bool __must_check current_clr_polling_and_test(void) -{ - __current_clr_polling(); - - /* - * Polling state must be visible before we test NEED_RESCHED, - * paired by resched_task() - */ - smp_mb(); - - return unlikely(tif_need_resched()); -} -#elif defined(TIF_POLLING_NRFLAG) +#ifdef TIF_POLLING_NRFLAG static inline int tsk_is_polling(struct task_struct *p) { return test_tsk_thread_flag(p, TIF_POLLING_NRFLAG); @@ -2762,7 +3059,7 @@ static inline bool __must_check current_set_polling_and_test(void) /* * Polling state must be visible before we test NEED_RESCHED, - * paired by resched_task() + * paired by resched_curr() */ smp_mb__after_atomic(); @@ -2780,7 +3077,7 @@ static inline bool __must_check current_clr_polling_and_test(void) /* * Polling state must be visible before we test NEED_RESCHED, - * paired by resched_task() + * paired by resched_curr() */ smp_mb__after_atomic(); @@ -2802,6 +3099,26 @@ static inline bool __must_check current_clr_polling_and_test(void) } #endif +static inline void current_clr_polling(void) +{ + __current_clr_polling(); + + /* + * Ensure we check TIF_NEED_RESCHED after we clear the polling bit. + * Once the bit is cleared, we'll get IPIs with every new + * TIF_NEED_RESCHED and the IPI handler, scheduler_ipi(), will also + * fold. + */ + smp_mb(); /* paired with resched_curr() */ + + preempt_fold_need_resched(); +} + +static __always_inline bool need_resched(void) +{ + return unlikely(tif_need_resched()); +} + /* * Thread group CPU time accounting. */ @@ -2843,6 +3160,11 @@ static inline unsigned int task_cpu(const struct task_struct *p) return task_thread_info(p)->cpu; } +static inline int task_node(const struct task_struct *p) +{ + return cpu_to_node(task_cpu(p)); +} + extern void set_task_cpu(struct task_struct *p, unsigned int cpu); #else @@ -2926,11 +3248,8 @@ extern void mm_init_owner(struct mm_struct *mm, struct task_struct *p); static inline void mm_update_next_owner(struct mm_struct *mm) { } - -static inline void mm_init_owner(struct mm_struct *mm, struct task_struct *p) -{ -} -#endif /* CONFIG_MM_OWNER */ +static inline void mm_init_owner(struct mm_struct *mm, struct task_struct *p) {} +#endif /* CONFIG_MEMCG */ static inline unsigned long task_rlimit(const struct task_struct *tsk, unsigned int limit) diff --git a/include/linux/sched/deadline.h b/include/linux/sched/deadline.h new file mode 100644 index 0000000000000..9d303b8847df2 --- /dev/null +++ b/include/linux/sched/deadline.h @@ -0,0 +1,24 @@ +#ifndef _SCHED_DEADLINE_H +#define _SCHED_DEADLINE_H + +/* + * SCHED_DEADLINE tasks has negative priorities, reflecting + * the fact that any of them has higher prio than RT and + * NORMAL/BATCH tasks. + */ + +#define MAX_DL_PRIO 0 + +static inline int dl_prio(int prio) +{ + if (unlikely(prio < MAX_DL_PRIO)) + return 1; + return 0; +} + +static inline int dl_task(struct task_struct *p) +{ + return dl_prio(p->prio); +} + +#endif /* _SCHED_DEADLINE_H */ diff --git a/include/linux/sched/prio.h b/include/linux/sched/prio.h new file mode 100644 index 0000000000000..d9cf5a5762d9d --- /dev/null +++ b/include/linux/sched/prio.h @@ -0,0 +1,60 @@ +#ifndef _SCHED_PRIO_H +#define _SCHED_PRIO_H + +#define MAX_NICE 19 +#define MIN_NICE -20 +#define NICE_WIDTH (MAX_NICE - MIN_NICE + 1) + +/* + * Priority of a process goes from 0..MAX_PRIO-1, valid RT + * priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL/SCHED_BATCH + * tasks are in the range MAX_RT_PRIO..MAX_PRIO-1. Priority + * values are inverted: lower p->prio value means higher priority. + * + * The MAX_USER_RT_PRIO value allows the actual maximum + * RT priority to be separate from the value exported to + * user-space. This allows kernel threads to set their + * priority to a value higher than any user task. Note: + * MAX_RT_PRIO must not be smaller than MAX_USER_RT_PRIO. + */ + +#define MAX_USER_RT_PRIO 100 +#define MAX_RT_PRIO MAX_USER_RT_PRIO + +#define MAX_PRIO (MAX_RT_PRIO + NICE_WIDTH) +#define DEFAULT_PRIO (MAX_RT_PRIO + NICE_WIDTH / 2) + +/* + * Convert user-nice values [ -20 ... 0 ... 19 ] + * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ], + * and back. + */ +#define NICE_TO_PRIO(nice) ((nice) + DEFAULT_PRIO) +#define PRIO_TO_NICE(prio) ((prio) - DEFAULT_PRIO) + +/* + * 'User priority' is the nice value converted to something we + * can work with better when scaling various scheduler parameters, + * it's a [ 0 ... 39 ] range. + */ +#define USER_PRIO(p) ((p)-MAX_RT_PRIO) +#define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio) +#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO)) + +/* + * Convert nice value [19,-20] to rlimit style value [1,40]. + */ +static inline long nice_to_rlimit(long nice) +{ + return (MAX_NICE - nice + 1); +} + +/* + * Convert rlimit style value [1,40] to nice value [-20, 19]. + */ +static inline long rlimit_to_nice(long prio) +{ + return (MAX_NICE - prio + 1); +} + +#endif /* _SCHED_PRIO_H */ diff --git a/include/linux/sched/rt.h b/include/linux/sched/rt.h index 440434df36274..5b85463773964 100644 --- a/include/linux/sched/rt.h +++ b/include/linux/sched/rt.h @@ -1,24 +1,7 @@ #ifndef _SCHED_RT_H #define _SCHED_RT_H -/* - * Priority of a process goes from 0..MAX_PRIO-1, valid RT - * priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL/SCHED_BATCH - * tasks are in the range MAX_RT_PRIO..MAX_PRIO-1. Priority - * values are inverted: lower p->prio value means higher priority. - * - * The MAX_USER_RT_PRIO value allows the actual maximum - * RT priority to be separate from the value exported to - * user-space. This allows kernel threads to set their - * priority to a value higher than any user task. Note: - * MAX_RT_PRIO must not be smaller than MAX_USER_RT_PRIO. - */ - -#define MAX_USER_RT_PRIO 100 -#define MAX_RT_PRIO MAX_USER_RT_PRIO - -#define MAX_PRIO (MAX_RT_PRIO + 40) -#define DEFAULT_PRIO (MAX_RT_PRIO + 20) +#include static inline int rt_prio(int prio) { @@ -34,7 +17,19 @@ static inline int rt_task(struct task_struct *p) #ifdef CONFIG_RT_MUTEXES extern int rt_mutex_getprio(struct task_struct *p); +extern int rt_mutex_getprio(struct task_struct *p); + extern void rt_mutex_setprio(struct task_struct *p, int prio); +extern int rt_mutex_get_effective_prio(struct task_struct *task, int newprio); +#if 0 +extern struct task_struct *rt_mutex_get_top_task(struct task_struct *task); +#else +// TJK: hack since CONFIG_RT_MUTEXES gets turned on implicitely +static inline struct task_struct *rt_mutex_get_top_task(struct task_struct *task) +{ + return NULL; +} +#endif extern void rt_mutex_adjust_pi(struct task_struct *p); static inline bool tsk_is_pi_blocked(struct task_struct *tsk) { @@ -45,6 +40,17 @@ static inline int rt_mutex_getprio(struct task_struct *p) { return p->normal_prio; } + +static inline int rt_mutex_get_effective_prio(struct task_struct *task, + int newprio) +{ + return newprio; +} + +static inline struct task_struct *rt_mutex_get_top_task(struct task_struct *task) +{ + return NULL; +} # define rt_mutex_adjust_pi(p) do { } while (0) static inline bool tsk_is_pi_blocked(struct task_struct *tsk) { diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 4d892dc630f75..aad9ce9ce47f0 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -2,10 +2,10 @@ #define _SCHED_SYSCTL_H #ifdef CONFIG_DETECT_HUNG_TASK +extern int sysctl_hung_task_check_count; extern unsigned int sysctl_hung_task_panic; -extern unsigned long sysctl_hung_task_check_count; extern unsigned long sysctl_hung_task_timeout_secs; -extern unsigned long sysctl_hung_task_warnings; +extern int sysctl_hung_task_warnings; extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); @@ -25,6 +25,10 @@ enum { sysctl_hung_task_timeout_secs = 0 }; * Because the kernel adds some informative sections to a image of program at * generating coredump, we need some margin. The number of extra sections is * 1-3 now and depends on arch. We use "5" as safe margin, here. + * + * ELF extended numbering allows more than 65535 sections, so 16-bit bound is + * not a hard limit any more. Although some userspace tools can be surprised by + * that. */ #define MAPCOUNT_ELF_CORE_MARGIN (5) #define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN) @@ -36,7 +40,7 @@ extern unsigned int sysctl_sched_min_granularity; extern unsigned int sysctl_sched_wakeup_granularity; extern unsigned int sysctl_sched_child_runs_first; extern unsigned int sysctl_sched_wake_to_idle; -extern unsigned int sysctl_sched_wakeup_load_threshold; +extern unsigned int sysctl_sched_capacity_margin; extern unsigned int sysctl_sched_window_stats_policy; extern unsigned int sysctl_sched_account_wait_time; extern unsigned int sysctl_sched_ravg_hist_size; @@ -44,11 +48,7 @@ extern unsigned int sysctl_sched_cpu_high_irqload; extern unsigned int sysctl_sched_freq_account_wait_time; extern unsigned int sysctl_sched_migration_fixup; extern unsigned int sysctl_sched_heavy_task_pct; -extern unsigned int sysctl_sched_min_runtime; extern unsigned int sysctl_sched_enable_power_aware; -#ifdef VENDOR_EDIT -extern unsigned int sysctl_thermal_aware_scheduling; -#endif #if defined(CONFIG_SCHED_FREQ_INPUT) || defined(CONFIG_SCHED_HMP) extern unsigned int sysctl_sched_init_task_load_pct; @@ -62,18 +62,32 @@ extern int sysctl_sched_freq_dec_notify; #ifdef CONFIG_SCHED_HMP extern unsigned int sysctl_sched_spill_nr_run; extern unsigned int sysctl_sched_spill_load_pct; -extern unsigned int sysctl_sched_small_task_pct; extern unsigned int sysctl_sched_upmigrate_pct; extern unsigned int sysctl_sched_downmigrate_pct; extern int sysctl_sched_upmigrate_min_nice; extern unsigned int sysctl_sched_powerband_limit_pct; extern unsigned int sysctl_sched_boost; +extern unsigned int sysctl_power_aware_timer_migration; +extern unsigned int sysctl_early_detection_duration; + +#ifdef CONFIG_SCHED_QHMP +extern unsigned int sysctl_sched_min_runtime; +extern unsigned int sysctl_sched_small_task_pct; +#else +extern unsigned int sysctl_sched_lowspill_freq; +extern unsigned int sysctl_sched_pack_freq; +#if defined(CONFIG_SCHED_FREQ_INPUT) +extern unsigned int sysctl_sched_new_task_windows; +#endif +#endif #else /* CONFIG_SCHED_HMP */ #define sysctl_sched_enable_hmp_task_placement 0 #endif /* CONFIG_SCHED_HMP */ +extern unsigned int sysctl_sched_is_big_little; +extern unsigned int sysctl_sched_sync_hint_enable; enum sched_tunable_scaling { SCHED_TUNABLESCALING_NONE, @@ -86,9 +100,7 @@ extern enum sched_tunable_scaling sysctl_sched_tunable_scaling; extern unsigned int sysctl_numa_balancing_scan_delay; extern unsigned int sysctl_numa_balancing_scan_period_min; extern unsigned int sysctl_numa_balancing_scan_period_max; -extern unsigned int sysctl_numa_balancing_scan_period_reset; extern unsigned int sysctl_numa_balancing_scan_size; -extern unsigned int sysctl_numa_balancing_settle_count; #ifdef CONFIG_SCHED_DEBUG extern unsigned int sysctl_sched_migration_cost; @@ -143,6 +155,15 @@ extern unsigned int sysctl_sched_cfs_bandwidth_slice; extern unsigned int sysctl_sched_autogroup_enabled; #endif +#ifdef CONFIG_SCHEDSTATS +extern unsigned int sysctl_sched_latency_panic_threshold; +extern unsigned int sysctl_sched_latency_warn_threshold; + +extern int sched_max_latency_sysctl(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos); +#endif + extern int sched_rr_timeslice; extern int sched_rr_handler(struct ctl_table *table, int write, @@ -153,4 +174,8 @@ extern int sched_rt_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); +extern int sysctl_numa_balancing(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos); + #endif /* _SCHED_SYSCTL_H */ diff --git a/include/linux/sched_energy.h b/include/linux/sched_energy.h new file mode 100644 index 0000000000000..a3f1627ac609e --- /dev/null +++ b/include/linux/sched_energy.h @@ -0,0 +1,36 @@ +#ifndef _LINUX_SCHED_ENERGY_H +#define _LINUX_SCHED_ENERGY_H + +#include +#include + +/* + * There doesn't seem to be an NR_CPUS style max number of sched domain + * levels so here's an arbitrary constant one for the moment. + * + * The levels alluded to here correspond to entries in struct + * sched_domain_topology_level that are meant to be populated by arch + * specific code (topology.c). + */ +#define NR_SD_LEVELS 8 + +#define SD_LEVEL0 0 +#define SD_LEVEL1 1 +#define SD_LEVEL2 2 +#define SD_LEVEL3 3 +#define SD_LEVEL4 4 +#define SD_LEVEL5 5 +#define SD_LEVEL6 6 +#define SD_LEVEL7 7 + +/* + * Convenience macro for iterating through said sd levels. + */ +#define for_each_possible_sd_level(level) \ + for (level = 0; level < NR_SD_LEVELS; level++) + +extern struct sched_group_energy *sge_array[NR_CPUS][NR_SD_LEVELS]; + +void init_sched_energy_costs(void); + +#endif diff --git a/include/linux/security.h b/include/linux/security.h index c1f7c807ae868..4b5df69059d53 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -1403,6 +1403,11 @@ static inline void security_free_mnt_opts(struct security_mnt_opts *opts) struct security_operations { char name[SECURITY_NAME_MAX + 1]; + int (*binder_set_context_mgr) (struct task_struct *mgr); + int (*binder_transaction) (struct task_struct *from, struct task_struct *to); + int (*binder_transfer_binder) (struct task_struct *from, struct task_struct *to); + int (*binder_transfer_file) (struct task_struct *from, struct task_struct *to, struct file *file); + int (*ptrace_access_check) (struct task_struct *child, unsigned int mode); int (*ptrace_traceme) (struct task_struct *parent); int (*capget) (struct task_struct *target, @@ -1695,6 +1700,10 @@ extern void __init security_fixup_ops(struct security_operations *ops); /* Security operations */ +int security_binder_set_context_mgr(struct task_struct *mgr); +int security_binder_transaction(struct task_struct *from, struct task_struct *to); +int security_binder_transfer_binder(struct task_struct *from, struct task_struct *to); +int security_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file); int security_ptrace_access_check(struct task_struct *child, unsigned int mode); int security_ptrace_traceme(struct task_struct *parent); int security_capget(struct task_struct *target, @@ -1879,6 +1888,26 @@ static inline int security_init(void) return 0; } +static inline int security_binder_set_context_mgr(struct task_struct *mgr) +{ + return 0; +} + +static inline int security_binder_transaction(struct task_struct *from, struct task_struct *to) +{ + return 0; +} + +static inline int security_binder_transfer_binder(struct task_struct *from, struct task_struct *to) +{ + return 0; +} + +static inline int security_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) +{ + return 0; +} + static inline int security_ptrace_access_check(struct task_struct *child, unsigned int mode) { diff --git a/include/linux/smp.h b/include/linux/smp.h index 868dcf0b2eb1a..c7d3d55419f2c 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -28,6 +28,7 @@ extern unsigned int total_cpus; int smp_call_function_single(int cpuid, smp_call_func_t func, void *info, int wait); +int smp_call_function_single_async(int cpuid, struct call_single_data *csd); #ifdef CONFIG_SMP diff --git a/include/linux/topology.h b/include/linux/topology.h index 9138f31200a4c..8f1c8c870c9f4 100644 --- a/include/linux/topology.h +++ b/include/linux/topology.h @@ -289,4 +289,9 @@ static inline int cpu_to_mem(int cpu) #define topology_core_cpumask(cpu) cpumask_of(cpu) #endif +static inline const struct cpumask *cpu_cpu_mask(int cpu) +{ + return cpumask_of_node(cpu_to_node(cpu)); +} + #endif /* _LINUX_TOPOLOGY_H */ diff --git a/include/linux/uidgid.h b/include/linux/uidgid.h index 5fc3c7f3bcc73..e7a8f1332f893 100755 --- a/include/linux/uidgid.h +++ b/include/linux/uidgid.h @@ -67,10 +67,6 @@ static inline gid_t __kgid_val(kgid_t gid) //huruihuan add for cgroup control #define GLOBAL_SYSTEM_UID KUIDT_INIT(1000) #endif - -#define GLOBAL_SYSTEM_UID KUIDT_INIT(1000) -#define GLOBAL_SYSTEM_GID KGIDT_INIT(1000) - #define INVALID_UID KUIDT_INIT(-1) #define INVALID_GID KGIDT_INIT(-1) diff --git a/include/trace/events/power.h b/include/trace/events/power.h index 65fe4ea8e3e01..47c34d93a85f1 100644 --- a/include/trace/events/power.h +++ b/include/trace/events/power.h @@ -539,6 +539,12 @@ DEFINE_EVENT(cpu_modes, cpu_mode_detect, mode, cpu_cnt) ); +DEFINE_EVENT(cpu, cpu_capacity, + + TP_PROTO(unsigned int capacity, unsigned int cpu_id), + TP_ARGS(capacity, cpu_id) +); + DECLARE_EVENT_CLASS(timer_status, TP_PROTO(unsigned int cpu, unsigned int single_enter_cycles, unsigned int single_enter_cycle_cnt, diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index 493bdf554bb4c..3162fc2ede02c 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -948,6 +948,390 @@ TRACE_EVENT(sched_get_nr_running_avg, __entry->avg, __entry->big_avg, __entry->iowait_avg) ); +/* + * Tracepoint for accounting sched averages for tasks. + */ +TRACE_EVENT(sched_load_avg_task, + + TP_PROTO(struct task_struct *tsk, struct sched_avg *avg), + + TP_ARGS(tsk, avg), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( int, cpu ) + __field( unsigned long, load ) + __field( unsigned long, utilization ) + __field( unsigned int, runnable_avg_sum ) + __field( unsigned int, running_avg_sum ) + __field( unsigned int, avg_period ) + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->cpu = task_cpu(tsk); + __entry->load = avg->load_avg_contrib; + __entry->utilization = avg->utilization_avg_contrib; + __entry->runnable_avg_sum = avg->runnable_avg_sum; + __entry->running_avg_sum = avg->running_avg_sum; + __entry->avg_period = avg->avg_period; + ), + + TP_printk("comm=%s pid=%d cpu=%d load=%lu utilization=%lu runnable_avg_sum=%u" + " running_avg_sum=%u avg_period=%u", + __entry->comm, __entry->pid, __entry->cpu, + __entry->load, __entry->utilization, + (unsigned int)__entry->runnable_avg_sum, + (unsigned int)__entry->running_avg_sum, + (unsigned int)__entry->avg_period) +); + +/* + * Tracepoint for accounting sched group energy + */ +TRACE_EVENT(sched_energy_diff, + + TP_PROTO(struct task_struct *tsk, int scpu, int dcpu, int udelta, + int nrgb, int nrga, int nrgd, int capb, int capa, int capd, + int nrgn, int nrgp), + + TP_ARGS(tsk, scpu, dcpu, udelta, + nrgb, nrga, nrgd, capb, capa, capd, + nrgn, nrgp), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( int, scpu ) + __field( int, dcpu ) + __field( int, udelta ) + __field( int, nrgb ) + __field( int, nrga ) + __field( int, nrgd ) + __field( int, capb ) + __field( int, capa ) + __field( int, capd ) + __field( int, nrgn ) + __field( int, nrgp ) + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->scpu = scpu; + __entry->dcpu = dcpu; + __entry->udelta = udelta; + __entry->nrgb = nrgb; + __entry->nrga = nrga; + __entry->nrgd = nrgd; + __entry->capb = capb; + __entry->capa = capa; + __entry->capd = capd; + __entry->nrgn = nrgn; + __entry->nrgp = nrgp; + ), + + TP_printk("pid=%d comm=%s " + "src_cpu=%d dst_cpu=%d usage_delta=%d " + "nrg_before=%d nrg_after=%d nrg_diff=%d " + "cap_before=%d cap_after=%d cap_delta=%d " + "nrg_delta=%d nrg_payoff=%d", + __entry->pid, __entry->comm, + __entry->scpu, __entry->dcpu, __entry->udelta, + __entry->nrgb, __entry->nrga, __entry->nrgd, + __entry->capb, __entry->capa, __entry->capd, + __entry->nrgn, __entry->nrgp) +); + + +/* + * Tracepoint for waking a polling cpu without an IPI. + */ +TRACE_EVENT(sched_wake_idle_without_ipi, + + TP_PROTO(int cpu), + + TP_ARGS(cpu), + + TP_STRUCT__entry( + __field( int, cpu ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + ), + + TP_printk("cpu=%d", __entry->cpu) +); + +TRACE_EVENT(sched_contrib_scale_f, + + TP_PROTO(int cpu, unsigned long freq_scale_factor, + unsigned long cpu_scale_factor), + + TP_ARGS(cpu, freq_scale_factor, cpu_scale_factor), + + TP_STRUCT__entry( + __field(int, cpu) + __field(unsigned long, freq_scale_factor) + __field(unsigned long, cpu_scale_factor) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->freq_scale_factor = freq_scale_factor; + __entry->cpu_scale_factor = cpu_scale_factor; + ), + + TP_printk("cpu=%d freq_scale_factor=%lu cpu_scale_factor=%lu", + __entry->cpu, __entry->freq_scale_factor, + __entry->cpu_scale_factor) +); + +TRACE_EVENT(sched_load_avg_cpu, + + TP_PROTO(int cpu, struct cfs_rq *cfs_rq), + + TP_ARGS(cpu, cfs_rq), + + TP_STRUCT__entry( + __field( int, cpu ) + __field( unsigned long, load ) + __field( unsigned long, utilization ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->load = cfs_rq->runnable_load_avg; + __entry->utilization = cfs_rq->utilization_load_avg; + ), + + TP_printk("cpu=%d load=%lu utilization=%lu", + __entry->cpu, __entry->load, __entry->utilization) +); + +/* + * Tracepoint for accounting task boosted utilization + */ +TRACE_EVENT(sched_boost_task, + + TP_PROTO(struct task_struct *tsk, unsigned long utilization, unsigned long margin), + + TP_ARGS(tsk, utilization, margin), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( unsigned long, utilization ) + __field( unsigned long, margin ) + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->utilization = utilization; + __entry->margin = margin; + ), + + TP_printk("comm=%s pid=%d utilization=%lu margin=%lu", + __entry->comm, __entry->pid, + __entry->utilization, + __entry->margin) +); + +TRACE_EVENT(sched_task_fits, + + TP_PROTO(struct task_struct *tsk, int cpu, int orig_usage, int usage, unsigned long task_util, unsigned long capacity, unsigned long margin), + + TP_ARGS(tsk, cpu, orig_usage, usage, task_util, capacity, margin), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( int, cpu ) + __field( int, orig_usage ) + __field( int, usage ) + __field( unsigned long, task_util ) + __field( unsigned long, capacity ) + __field( unsigned long, margin ) + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->cpu = cpu; + __entry->orig_usage = orig_usage; + __entry->usage = usage; + __entry->capacity = capacity; + __entry->task_util = task_util; + __entry->margin = margin; + ), + + TP_printk("comm=%s pid=%d cpu=%d origu=%d tasku=%ld u=%d boost=%d capacity=%lu margin=%lu result=%d", + __entry->comm, __entry->pid, + __entry->cpu, + __entry->orig_usage, + __entry->task_util, + __entry->usage, + __entry->usage-__entry->orig_usage, + __entry->capacity, + __entry->margin, + (__entry->capacity * 1024) > (__entry->usage * __entry->margin) + ) +); + +/* + * Tracepoint for accounting CPU boosted utilization + */ +TRACE_EVENT(sched_boost_cpu, + + TP_PROTO(int cpu, unsigned long usage, unsigned long margin), + + TP_ARGS(cpu, usage, margin), + + TP_STRUCT__entry( + __field( int, cpu ) + __field( unsigned long, usage ) + __field( unsigned long, margin ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->usage = usage; + __entry->margin = margin; + ), + + TP_printk("cpu=%d usage=%lu margin=%lu", + __entry->cpu, + __entry->usage, + __entry->margin) +); + +/* + * Tracepoint for schedtune_tasks_update + */ +TRACE_EVENT(sched_tune_filter, + + TP_PROTO(int nrg_delta, int cap_delta, int nrg_payoff, int region), + + TP_ARGS(nrg_delta, cap_delta, nrg_payoff, region), + + TP_STRUCT__entry( + __field( int, nrg_delta ) + __field( int, cap_delta ) + __field( int, nrg_payoff ) + __field( int, region ) + ), + + TP_fast_assign( + __entry->nrg_delta = nrg_delta; + __entry->cap_delta = cap_delta; + __entry->nrg_payoff = nrg_payoff; + __entry->region = region; + ), + + TP_printk("nrg_delta=%d cap_delta=%d nrg_payoff=%d region=%d", + __entry->nrg_delta, __entry->cap_delta, + __entry->nrg_payoff, __entry->region) +); + +/* + * Tracepoint for schedtune_boostgroup_update + */ +TRACE_EVENT(sched_tune_boostgroup_update, + + TP_PROTO(int cpu, int variation, int max_boost), + + TP_ARGS(cpu, variation, max_boost), + + TP_STRUCT__entry( + __field( int, cpu ) + __field( int, variation ) + __field( int, max_boost ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->variation = variation; + __entry->max_boost = max_boost; + ), + + TP_printk("cpu=%d variation=%d max_boost=%d", + __entry->cpu, __entry->variation, __entry->max_boost) +); + +/* + * Tracepoint for schedtune_tasks_update + */ +TRACE_EVENT(sched_tune_tasks_update, + + TP_PROTO(struct task_struct *tsk, int cpu, int tasks, int idx, + unsigned int boost, unsigned int max_boost), + + TP_ARGS(tsk, cpu, tasks, idx, boost, max_boost), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( int, cpu ) + __field( int, tasks ) + __field( int, idx ) + __field( unsigned int, boost ) + __field( unsigned int, max_boost ) + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->cpu = cpu; + __entry->tasks = tasks; + __entry->idx = idx; + __entry->boost = boost; + __entry->max_boost = max_boost; + ), + + TP_printk("pid=%d comm=%s " + "cpu=%d tasks=%d idx=%d boost=%u max_boost=%u", + __entry->pid, __entry->comm, + __entry->cpu, __entry->tasks, __entry->idx, + __entry->boost, __entry->max_boost) +); + +/* + * Tracepoint for sched_tune_config settings + */ +TRACE_EVENT(sched_tune_config, + + TP_PROTO(int boost, int pb_nrg_gain, int pb_cap_gain, int pc_nrg_gain, int pc_cap_gain), + + TP_ARGS(boost, pb_nrg_gain, pb_cap_gain, pc_nrg_gain, pc_cap_gain), + + TP_STRUCT__entry( + __field( int, boost ) + __field( int, pb_nrg_gain ) + __field( int, pb_cap_gain ) + __field( int, pc_nrg_gain ) + __field( int, pc_cap_gain ) + ), + + TP_fast_assign( + __entry->boost = boost; + __entry->pb_nrg_gain = pb_nrg_gain; + __entry->pb_cap_gain = pb_cap_gain; + __entry->pc_nrg_gain = pc_nrg_gain; + __entry->pc_cap_gain = pc_cap_gain; + ), + + TP_printk("boost=%d " + "pb_nrg_gain=%d pb_cap_gain=%d " + "pc_nrg_gain=%d pc_cap_gain=%d", + __entry->boost, + __entry->pb_nrg_gain, __entry->pb_cap_gain, + __entry->pc_nrg_gain, __entry->pc_cap_gain) +); + #endif /* _TRACE_SCHED_H */ /* This part must be outside protection */ diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h index 5a0f945927ace..cf1a839091117 100644 --- a/include/uapi/linux/sched.h +++ b/include/uapi/linux/sched.h @@ -39,8 +39,10 @@ #define SCHED_BATCH 3 /* SCHED_ISO: reserved but not implemented yet */ #define SCHED_IDLE 5 +#define SCHED_DEADLINE 6 /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ #define SCHED_RESET_ON_FORK 0x40000000 +#define SCHED_FLAG_RESET_ON_FORK 0x01 #endif /* _UAPI_LINUX_SCHED_H */ diff --git a/init/Kconfig b/init/Kconfig index c353169a708fe..8fba3ca0bd3b6 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -867,6 +867,24 @@ config RESOURCE_COUNTERS This option enables controller independent resource accounting infrastructure that works with cgroups. +config CGROUP_SCHEDTUNE + bool "Task boosting cgroup subsystem for EAS (EXPERIMENTAL)" + depends on SCHED_TUNE + help + This option provides the "schedtune" controller which improve the + flexibility of the task boosting mechanism by introducing the support + to define "per task" boost values. + + This new controller: + 1. allows only a two layers hierarchy, where the root defines the + system-wide boost value and its direct childs define each one a + different "class of tasks" to be boosted with a different value + 2. supports up to 16 different task classes, each one which could be + configured with a different boost value + + Only if you are testing a kernel with energy-aware scheduler + support, you might want to say Y here. + config MEMCG bool "Memory Resource Controller for Control Groups" depends on RESOURCE_COUNTERS @@ -1143,6 +1161,33 @@ config SCHED_AUTOGROUP desktop applications. Task group autogeneration is currently based upon task session. +config SCHED_TUNE + bool "Tasks boosting for energy-aware scheduler (EXPERIMENTAL)" + help + This option enable the system-wide support for task boosting. + When this support is enabled a new sysctl interface is exposed to + userspace via: + /proc/sys/kernel/sched_cfs_boost + which allows to set a system-wide boost value in range [0..100]. + + The currently boosting strategy is implemented in such a way that: + - a 0% boost value requires to operate in "standard" EAS mode by + scheduling all tasks at the minimum capacities required by their + workload demand + - a 100% boost value requires to push at maximum the task + performances, "regardless" of the incurred energy consumption + + A boost value in between these two boundaries is used to bias the + power/performance trade-off, the higher the boost value the more the + EAS scheduler is biased toward performance boosting instead of energy + efficiency. + + Since this support exposes a single system-wide knob, the specified + boost value is applied to all (CFS) tasks in the system. + + Only if you are testing a kernel with energy-aware scheduler support, + you might want to say Y here. + config MM_OWNER bool diff --git a/init/calibrate.c b/init/calibrate.c index fda0a7b0f06c0..fe85c605c7d7b 100644 --- a/init/calibrate.c +++ b/init/calibrate.c @@ -31,7 +31,7 @@ __setup("lpj=", lpj_setup); #define DELAY_CALIBRATION_TICKS ((HZ < 100) ? 1 : (HZ/100)) #define MAX_DIRECT_CALIBRATION_RETRIES 5 -static unsigned long __cpuinit calibrate_delay_direct(void) +static unsigned long calibrate_delay_direct(void) { unsigned long pre_start, start, post_start; unsigned long pre_end, end, post_end; @@ -166,7 +166,7 @@ static unsigned long __cpuinit calibrate_delay_direct(void) return 0; } #else -static unsigned long __cpuinit calibrate_delay_direct(void) {return 0;} +static unsigned long calibrate_delay_direct(void) {return 0;} #endif /* @@ -180,7 +180,7 @@ static unsigned long __cpuinit calibrate_delay_direct(void) {return 0;} */ #define LPS_PREC 8 -static unsigned long __cpuinit calibrate_delay_converge(void) +static unsigned long calibrate_delay_converge(void) { /* First stage - slowly accelerate to find initial bounds */ unsigned long lpj, lpj_base, ticks, loopadd, loopadd_base, chop_limit; @@ -254,12 +254,12 @@ static DEFINE_PER_CPU(unsigned long, cpu_loops_per_jiffy) = { 0 }; * Architectures should override this function if a faster calibration * method is available. */ -unsigned long __attribute__((weak)) __cpuinit calibrate_delay_is_known(void) +unsigned long __attribute__((weak)) calibrate_delay_is_known(void) { return 0; } -void __cpuinit calibrate_delay(void) +void calibrate_delay(void) { unsigned long lpj; static bool printed; diff --git a/kernel/Makefile b/kernel/Makefile index 99d1fec9da3dc..fac636723f59f 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -7,7 +7,7 @@ obj-y = fork.o exec_domain.o panic.o printk.o \ sysctl.o sysctl_binary.o capability.o ptrace.o timer.o user.o \ signal.o sys.o kmod.o workqueue.o pid.o task_work.o \ rcupdate.o extable.o params.o posix-timers.o \ - kthread.o wait.o sys_ni.o posix-cpu-timers.o mutex.o \ + kthread.o sys_ni.o posix-cpu-timers.o mutex.o \ hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \ notifier.o ksysfs.o cred.o \ async.o range.o groups.o lglock.o smpboot.o diff --git a/kernel/cgroup.c b/kernel/cgroup.c index 1f59aaa60497a..f381a941492a2 100755 --- a/kernel/cgroup.c +++ b/kernel/cgroup.c @@ -2155,9 +2155,7 @@ static int attach_task_by_pid(struct cgroup *cgrp, u64 pid, bool threadgroup) const struct cred *cred = current_cred(), *tcred; int ret; - skip_cfs_throttle(1); if (!cgroup_lock_live_group(cgrp)) { - skip_cfs_throttle(0); return -ENODEV; } @@ -2235,7 +2233,6 @@ static int attach_task_by_pid(struct cgroup *cgrp, u64 pid, bool threadgroup) put_task_struct(tsk); out_unlock_cgroup: mutex_unlock(&cgroup_mutex); - skip_cfs_throttle(0); return ret; } diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 383f8231e4367..7be0071adc149 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -70,6 +70,7 @@ void user_enter(void) local_irq_restore(flags); } +#ifdef TJK_HMP #ifdef CONFIG_PREEMPT /** * preempt_schedule_context - preempt_schedule called by tracing @@ -110,6 +111,7 @@ void __sched notrace preempt_schedule_context(void) } EXPORT_SYMBOL_GPL(preempt_schedule_context); #endif /* CONFIG_PREEMPT */ +#endif /** * user_exit - Inform the context tracking that the CPU is diff --git a/kernel/cpu.c b/kernel/cpu.c index 5ac6ff97a98b2..575906b0ea03c 100755 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -384,7 +384,7 @@ EXPORT_SYMBOL(cpu_down); #endif /*CONFIG_HOTPLUG_CPU*/ /* Requires cpu_add_remove_lock to be held */ -static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen) +static int _cpu_up(unsigned int cpu, int tasks_frozen) { int ret, nr_calls = 0; void *hcpu = (void *)(long)cpu; @@ -438,7 +438,7 @@ static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen) return ret; } -int __cpuinit cpu_up(unsigned int cpu) +int cpu_up(unsigned int cpu) { int err = 0; @@ -636,7 +636,7 @@ core_initcall(cpu_hotplug_pm_sync_init); * It must be called by the arch code on the new cpu, before the new cpu * enables interrupts and before the "boot" cpu returns from __cpu_up(). */ -void __cpuinit notify_cpu_starting(unsigned int cpu) +void notify_cpu_starting(unsigned int cpu) { unsigned long val = CPU_STARTING; diff --git a/kernel/events/core.c b/kernel/events/core.c index 9dbed267efb5a..bb0fb2f862f5e 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7867,7 +7867,7 @@ static void __init perf_event_init_all_cpus(void) } } -static void __cpuinit perf_event_init_cpu(int cpu) +static void perf_event_init_cpu(int cpu) { struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); @@ -7999,7 +7999,7 @@ static struct notifier_block perf_reboot_notifier = { .priority = INT_MIN, }; -static int __cpuinit +static int perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (long)hcpu; diff --git a/kernel/exit.c b/kernel/exit.c index e58c525cbc3ab..540bad43812df 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -777,10 +777,6 @@ void do_exit(long code) sched_exit(tsk); - if (tsk->flags & PF_SU) { - su_exit(); - } - /* * tsk->flags are checked in the futex code to protect against * an exiting task cleaning up the robust pi futexes. diff --git a/kernel/fork.c b/kernel/fork.c index 4adf3bb29d2a1..aece712421540 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -326,8 +326,6 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) if (err) goto free_ti; - tsk->flags &= ~PF_SU; - tsk->stack = ti; #ifdef CONFIG_SECCOMP /* @@ -1375,7 +1373,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, #endif /* Perform scheduler related setup. Assign this task to a CPU. */ - sched_fork(p); + sched_fork(clone_flags, p); retval = perf_event_init_task(p); if (retval) @@ -1617,7 +1615,7 @@ static inline void init_idle_pids(struct pid_link *links) } } -struct task_struct * __cpuinit fork_idle(int cpu) +struct task_struct * fork_idle(int cpu) { struct task_struct *task; task = copy_process(CLONE_VM, 0, 0, NULL, &init_struct_pid, 0); diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index a5f4dfeeab86b..17b82e5812fb7 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -1685,7 +1685,7 @@ SYSCALL_DEFINE2(nanosleep, struct timespec __user *, rqtp, /* * Functions related to boot-time initialization: */ -static void __cpuinit init_hrtimers_cpu(int cpu) +static void init_hrtimers_cpu(int cpu) { struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); int i; @@ -1766,7 +1766,7 @@ static void migrate_hrtimers(int scpu) #endif /* CONFIG_HOTPLUG_CPU */ -static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self, +static int hrtimer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { int scpu = (long)hcpu; @@ -1799,7 +1799,7 @@ static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata hrtimers_nb = { +static struct notifier_block hrtimers_nb = { .notifier_call = hrtimer_cpu_notify, }; diff --git a/kernel/kthread.c b/kernel/kthread.c index c56c6f8ec6078..2ccd5dcfdd310 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -547,6 +547,19 @@ int kthread_worker_fn(void *worker_ptr) } EXPORT_SYMBOL_GPL(kthread_worker_fn); +/* + * Returns true when the work could not be queued at the moment. + * It happens when it is already pending in a worker list + * or when it is being cancelled. + */ +static inline bool queuing_blocked(struct kthread_worker *worker, + struct kthread_work *work) +{ + lockdep_assert_held(&worker->lock); + + return !list_empty(&work->node) || work->canceling; +} + /* insert @work before @pos in @worker */ static void insert_kthread_work(struct kthread_worker *worker, struct kthread_work *work, @@ -576,7 +589,7 @@ bool queue_kthread_work(struct kthread_worker *worker, unsigned long flags; spin_lock_irqsave(&worker->lock, flags); - if (list_empty(&work->node)) { + if (!queuing_blocked(worker, work)) { insert_kthread_work(worker, work, &worker->work_list); ret = true; } @@ -637,6 +650,87 @@ void flush_kthread_work(struct kthread_work *work) } EXPORT_SYMBOL_GPL(flush_kthread_work); +/* + * This function removes the work from the worker queue. Also it makes sure + * that it won't get queued later via the delayed work's timer. + * + * The work might still be in use when this function finishes. See the + * current_work proceed by the worker. + * + * Return: %true if @work was pending and successfully canceled, + * %false if @work was not pending + */ +static bool __kthread_cancel_work(struct kthread_work *work, + unsigned long *flags) +{ + /* + * Try to remove the work from a worker list. It might either + * be from worker->work_list or from worker->delayed_work_list. + */ + if (!list_empty(&work->node)) { + list_del_init(&work->node); + return true; + } + + return false; +} + +static bool __kthread_cancel_work_sync(struct kthread_work *work) +{ + struct kthread_worker *worker = work->worker; + unsigned long flags; + int ret = false; + + if (!worker) + goto out; + + spin_lock_irqsave(&worker->lock, flags); + /* Work must not be used with >1 worker, see kthread_queue_work(). */ + WARN_ON_ONCE(work->worker != worker); + + ret = __kthread_cancel_work(work, &flags); + + if (worker->current_work != work) + goto out_fast; + + /* + * The work is in progress and we need to wait with the lock released. + * In the meantime, block any queuing by setting the canceling counter. + */ + work->canceling++; + spin_unlock_irqrestore(&worker->lock, flags); + flush_kthread_work(work); + spin_lock_irqsave(&worker->lock, flags); + work->canceling--; + +out_fast: + spin_unlock_irqrestore(&worker->lock, flags); +out: + return ret; +} + +/** + * kthread_cancel_work_sync - cancel a kthread work and wait for it to finish + * @work: the kthread work to cancel + * + * Cancel @work and wait for its execution to finish. This function + * can be used even if the work re-queues itself. On return from this + * function, @work is guaranteed to be not pending or executing on any CPU. + * + * kthread_cancel_work_sync(&delayed_work->work) must not be used for + * delayed_work's. Use kthread_cancel_delayed_work_sync() instead. + * + * The caller must ensure that the worker on which @work was last + * queued can't be destroyed before this function returns. + * + * Return: %true if @work was pending, %false otherwise. + */ +bool kthread_cancel_work_sync(struct kthread_work *work) +{ + return __kthread_cancel_work_sync(work); +} +EXPORT_SYMBOL_GPL(kthread_cancel_work_sync); + /** * flush_kthread_worker - flush all current works on a kthread_worker * @worker: worker to flush diff --git a/kernel/printk.c b/kernel/printk.c index d737ae0bc8a0d..99823f8057c33 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -2310,13 +2310,13 @@ void resume_console(void) console_unlock(); } -static void __cpuinit console_flush(struct work_struct *work) +static void console_flush(struct work_struct *work) { console_lock(); console_unlock(); } -static __cpuinitdata DECLARE_WORK(console_cpu_notify_work, console_flush); +static DECLARE_WORK(console_cpu_notify_work, console_flush); /** * console_cpu_notify - print deferred console messages after CPU hotplug @@ -2332,7 +2332,7 @@ static __cpuinitdata DECLARE_WORK(console_cpu_notify_work, console_flush); * Special handling must be done for cases invoked from an atomic context, * as we can't be taking the console semaphore here. */ -static int __cpuinit console_cpu_notify(struct notifier_block *self, +static int console_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { switch (action) { diff --git a/kernel/profile.c b/kernel/profile.c index 594dff58c6078..4c0cb955f33c9 100644 --- a/kernel/profile.c +++ b/kernel/profile.c @@ -331,7 +331,7 @@ static void do_profile_hits(int type, void *__pc, unsigned int nr_hits) put_cpu(); } -static int __cpuinit profile_cpu_callback(struct notifier_block *info, +static int profile_cpu_callback(struct notifier_block *info, unsigned long action, void *__cpu) { int node, cpu = (unsigned long)__cpu; diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c index e1f3a8c967243..ceaa5d55671fc 100644 --- a/kernel/rcutorture.c +++ b/kernel/rcutorture.c @@ -1514,7 +1514,7 @@ rcu_torture_shutdown(void *arg) * Execute random CPU-hotplug operations at the interval specified * by the onoff_interval. */ -static int __cpuinit +static int rcu_torture_onoff(void *arg) { int cpu; @@ -1596,7 +1596,7 @@ rcu_torture_onoff(void *arg) return 0; } -static int __cpuinit +static int rcu_torture_onoff_init(void) { int ret; @@ -1639,7 +1639,7 @@ static void rcu_torture_onoff_cleanup(void) * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then * induces a CPU stall for the time specified by stall_cpu. */ -static int __cpuinit rcu_torture_stall(void *args) +static int rcu_torture_stall(void *args) { unsigned long stop_at; diff --git a/kernel/rcutree.c b/kernel/rcutree.c index c5cc4f1bce80f..20a56ced31015 100644 --- a/kernel/rcutree.c +++ b/kernel/rcutree.c @@ -2996,7 +2996,7 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp) * can accept some slop in the rsp->completed access due to the fact * that this CPU cannot possibly have any RCU callbacks in flight yet. */ -static void __cpuinit +static void rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible) { unsigned long flags; @@ -3049,7 +3049,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible) mutex_unlock(&rsp->onoff_mutex); } -static void __cpuinit rcu_prepare_cpu(int cpu) +static void rcu_prepare_cpu(int cpu) { struct rcu_state *rsp; @@ -3061,7 +3061,7 @@ static void __cpuinit rcu_prepare_cpu(int cpu) /* * Handle CPU online/offline notification events. */ -static int __cpuinit rcu_cpu_notify(struct notifier_block *self, +static int rcu_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { long cpu = (long)hcpu; diff --git a/kernel/rcutree.h b/kernel/rcutree.h index b07ec0473883d..f362084a9c90b 100644 --- a/kernel/rcutree.h +++ b/kernel/rcutree.h @@ -523,10 +523,10 @@ static void invoke_rcu_callbacks_kthread(void); static bool rcu_is_callbacks_kthread(void); #ifdef CONFIG_RCU_BOOST static void rcu_preempt_do_callbacks(void); -static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp, +static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp, struct rcu_node *rnp); #endif /* #ifdef CONFIG_RCU_BOOST */ -static void __cpuinit rcu_prepare_kthreads(int cpu); +static void rcu_prepare_kthreads(int cpu); static void rcu_prepare_for_idle_init(int cpu); static void rcu_cleanup_after_idle(int cpu); static void rcu_prepare_for_idle(int cpu); diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h index 104bb6b314f78..eb0adea7e0147 100644 --- a/kernel/rcutree_plugin.h +++ b/kernel/rcutree_plugin.h @@ -1327,7 +1327,7 @@ static void rcu_preempt_boost_start_gp(struct rcu_node *rnp) * already exist. We only create this kthread for preemptible RCU. * Returns zero if all is well, a negated errno otherwise. */ -static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp, +static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp, struct rcu_node *rnp) { int rnp_index = rnp - &rsp->node[0]; @@ -1482,7 +1482,7 @@ static int __init rcu_spawn_kthreads(void) } early_initcall(rcu_spawn_kthreads); -static void __cpuinit rcu_prepare_kthreads(int cpu) +static void rcu_prepare_kthreads(int cpu) { struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, cpu); struct rcu_node *rnp = rdp->mynode; @@ -1524,7 +1524,7 @@ static int __init rcu_scheduler_really_started(void) } early_initcall(rcu_scheduler_really_started); -static void __cpuinit rcu_prepare_kthreads(int cpu) +static void rcu_prepare_kthreads(int cpu) { } diff --git a/kernel/relay.c b/kernel/relay.c index b91488ba2e5a7..5001c9887db1a 100644 --- a/kernel/relay.c +++ b/kernel/relay.c @@ -516,7 +516,7 @@ static void setup_callbacks(struct rchan *chan, * * Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD) */ -static int __cpuinit relay_hotcpu_callback(struct notifier_block *nb, +static int relay_hotcpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu) { diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 1c7bde63c5e5b..5a35ff5eebff5 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -11,9 +11,13 @@ ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer endif -obj-y += core.o clock.o cputime.o idle_task.o fair.o rt.o stop_task.o sched_avg.o -obj-$(CONFIG_SMP) += cpupri.o +obj-y += core.o loadavg.o clock.o cputime.o +obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o energy.o +obj-y += wait.o completion.o +obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o obj-$(CONFIG_SCHEDSTATS) += stats.o obj-$(CONFIG_SCHED_DEBUG) += debug.o +obj-$(CONFIG_SCHED_TUNE) += tune.o obj-$(CONFIG_CGROUP_CPUACCT) += cpuacct.o +obj-$(CONFIG_CPU_FREQ_GOV_SCHED) += cpufreq_sched.o diff --git a/kernel/sched/auto_group.c b/kernel/sched/auto_group.c index 4a073539c58e6..eae160dd669d9 100644 --- a/kernel/sched/auto_group.c +++ b/kernel/sched/auto_group.c @@ -87,8 +87,7 @@ static inline struct autogroup *autogroup_create(void) * so we don't have to move tasks around upon policy change, * or flail around trying to allocate bandwidth on the fly. * A bandwidth exception in __sched_setscheduler() allows - * the policy change to proceed. Thereafter, task_group() - * returns &root_task_group, so zero bandwidth is required. + * the policy change to proceed. */ free_rt_sched_group(tg); tg->rt_se = root_task_group.rt_se; @@ -115,9 +114,6 @@ bool task_wants_autogroup(struct task_struct *p, struct task_group *tg) if (tg != &root_task_group) return false; - if (p->sched_class != &fair_sched_class) - return false; - /* * We can only assume the task group can't go away on us if * autogroup_move_group() can see us on ->thread_group list. @@ -148,11 +144,8 @@ autogroup_move_group(struct task_struct *p, struct autogroup *ag) if (!ACCESS_ONCE(sysctl_sched_autogroup_enabled)) goto out; - t = p; - do { + for_each_thread(p, t) sched_move_task(t); - } while_each_thread(p, t); - out: unlock_task_sighand(p, &flags); autogroup_kref_put(prev); @@ -203,7 +196,7 @@ int proc_sched_autogroup_set_nice(struct task_struct *p, int nice) struct autogroup *ag; int err; - if (nice < -20 || nice > 19) + if (nice < MIN_NICE || nice > MAX_NICE) return -EINVAL; err = security_task_setnice(current, nice); diff --git a/kernel/sched/clock.c b/kernel/sched/clock.c index c3ae1446461c0..c27e4f8f4879e 100644 --- a/kernel/sched/clock.c +++ b/kernel/sched/clock.c @@ -26,9 +26,10 @@ * at 0 on boot (but people really shouldn't rely on that). * * cpu_clock(i) -- can be used from any context, including NMI. - * sched_clock_cpu(i) -- must be used with local IRQs disabled (implied by NMI) * local_clock() -- is cpu_clock() on the current cpu. * + * sched_clock_cpu(i) + * * How: * * The implementation either uses sched_clock() when @@ -50,15 +51,6 @@ * Furthermore, explicit sleep and wakeup hooks allow us to account for time * that is otherwise invisible (TSC gets stopped). * - * - * Notes: - * - * The !IRQ-safetly of sched_clock() and sched_clock_cpu() comes from things - * like cpufreq interrupts that can change the base clock (TSC) multiplier - * and cause funny jumps in time -- although the filtering provided by - * sched_clock_cpu() should mitigate serious artifacts we cannot rely on it - * in general since for !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK we fully rely on - * sched_clock(). */ #include #include @@ -66,13 +58,16 @@ #include #include #include +#include +#include +#include /* * Scheduler clock - returns current time in nanosec units. * This is default implementation. * Architectures and sub-architectures can override this. */ -unsigned long long __attribute__((weak)) sched_clock(void) +unsigned long long __weak sched_clock(void) { return (unsigned long long)(jiffies - INITIAL_JIFFIES) * (NSEC_PER_SEC / HZ); @@ -82,7 +77,52 @@ EXPORT_SYMBOL_GPL(sched_clock); __read_mostly int sched_clock_running; #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK -__read_mostly int sched_clock_stable; +static struct static_key __sched_clock_stable = STATIC_KEY_INIT; +static int __sched_clock_stable_early; + +int sched_clock_stable(void) +{ + return static_key_false(&__sched_clock_stable); +} + +static void __set_sched_clock_stable(void) +{ + if (!sched_clock_stable()) + static_key_slow_inc(&__sched_clock_stable); +} + +void set_sched_clock_stable(void) +{ + __sched_clock_stable_early = 1; + + smp_mb(); /* matches sched_clock_init() */ + + if (!sched_clock_running) + return; + + __set_sched_clock_stable(); +} + +static void __clear_sched_clock_stable(struct work_struct *work) +{ + /* XXX worry about clock continuity */ + if (sched_clock_stable()) + static_key_slow_dec(&__sched_clock_stable); +} + +static DECLARE_WORK(sched_clock_work, __clear_sched_clock_stable); + +void clear_sched_clock_stable(void) +{ + __sched_clock_stable_early = 0; + + smp_mb(); /* matches sched_clock_init() */ + + if (!sched_clock_running) + return; + + schedule_work(&sched_clock_work); +} struct sched_clock_data { u64 tick_raw; @@ -94,7 +134,7 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); static inline struct sched_clock_data *this_scd(void) { - return &__get_cpu_var(sched_clock_data); + return this_cpu_ptr(&sched_clock_data); } static inline struct sched_clock_data *cpu_sdc(int cpu) @@ -116,6 +156,20 @@ void sched_clock_init(void) } sched_clock_running = 1; + + /* + * Ensure that it is impossible to not do a static_key update. + * + * Either {set,clear}_sched_clock_stable() must see sched_clock_running + * and do the update, or we must see their __sched_clock_stable_early + * and do the update, or both. + */ + smp_mb(); /* matches {set,clear}_sched_clock_stable() */ + + if (__sched_clock_stable_early) + __set_sched_clock_stable(); + else + __clear_sched_clock_stable(NULL); } /* @@ -242,20 +296,20 @@ u64 sched_clock_cpu(int cpu) struct sched_clock_data *scd; u64 clock; - WARN_ON_ONCE(!irqs_disabled()); - - if (sched_clock_stable) + if (sched_clock_stable()) return sched_clock(); if (unlikely(!sched_clock_running)) return 0ull; + preempt_disable_notrace(); scd = cpu_sdc(cpu); if (cpu != smp_processor_id()) clock = sched_clock_remote(scd); else clock = sched_clock_local(scd); + preempt_enable_notrace(); return clock; } @@ -265,7 +319,7 @@ void sched_clock_tick(void) struct sched_clock_data *scd; u64 now, now_gtod; - if (sched_clock_stable) + if (sched_clock_stable()) return; if (unlikely(!sched_clock_running)) @@ -316,14 +370,10 @@ EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); */ u64 cpu_clock(int cpu) { - u64 clock; - unsigned long flags; - - local_irq_save(flags); - clock = sched_clock_cpu(cpu); - local_irq_restore(flags); + if (!sched_clock_stable()) + return sched_clock_cpu(cpu); - return clock; + return sched_clock(); } /* @@ -335,14 +385,10 @@ u64 cpu_clock(int cpu) */ u64 local_clock(void) { - u64 clock; - unsigned long flags; + if (!sched_clock_stable()) + return sched_clock_cpu(raw_smp_processor_id()); - local_irq_save(flags); - clock = sched_clock_cpu(smp_processor_id()); - local_irq_restore(flags); - - return clock; + return sched_clock(); } #else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ @@ -362,12 +408,12 @@ u64 sched_clock_cpu(int cpu) u64 cpu_clock(int cpu) { - return sched_clock_cpu(cpu); + return sched_clock(); } u64 local_clock(void) { - return sched_clock_cpu(0); + return sched_clock(); } #endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ diff --git a/kernel/sched/completion.c b/kernel/sched/completion.c new file mode 100644 index 0000000000000..a63f4dc27909e --- /dev/null +++ b/kernel/sched/completion.c @@ -0,0 +1,299 @@ +/* + * Generic wait-for-completion handler; + * + * It differs from semaphores in that their default case is the opposite, + * wait_for_completion default blocks whereas semaphore default non-block. The + * interface also makes it easy to 'complete' multiple waiting threads, + * something which isn't entirely natural for semaphores. + * + * But more importantly, the primitive documents the usage. Semaphores would + * typically be used for exclusion which gives rise to priority inversion. + * Waiting for completion is a typically sync point, but not an exclusion point. + */ + +#include +#include + +/** + * complete: - signals a single thread waiting on this completion + * @x: holds the state of this particular completion + * + * This will wake up a single thread waiting on this completion. Threads will be + * awakened in the same order in which they were queued. + * + * See also complete_all(), wait_for_completion() and related routines. + * + * It may be assumed that this function implies a write memory barrier before + * changing the task state if and only if any tasks are woken up. + */ +void complete(struct completion *x) +{ + unsigned long flags; + + spin_lock_irqsave(&x->wait.lock, flags); + x->done++; + __wake_up_locked(&x->wait, TASK_NORMAL, 1); + spin_unlock_irqrestore(&x->wait.lock, flags); +} +EXPORT_SYMBOL(complete); + +/** + * complete_all: - signals all threads waiting on this completion + * @x: holds the state of this particular completion + * + * This will wake up all threads waiting on this particular completion event. + * + * It may be assumed that this function implies a write memory barrier before + * changing the task state if and only if any tasks are woken up. + */ +void complete_all(struct completion *x) +{ + unsigned long flags; + + spin_lock_irqsave(&x->wait.lock, flags); + x->done += UINT_MAX/2; + __wake_up_locked(&x->wait, TASK_NORMAL, 0); + spin_unlock_irqrestore(&x->wait.lock, flags); +} +EXPORT_SYMBOL(complete_all); + +static inline long __sched +do_wait_for_common(struct completion *x, + long (*action)(long), long timeout, int state) +{ + if (!x->done) { + DECLARE_WAITQUEUE(wait, current); + + __add_wait_queue_tail_exclusive(&x->wait, &wait); + do { + if (signal_pending_state(state, current)) { + timeout = -ERESTARTSYS; + break; + } + __set_current_state(state); + spin_unlock_irq(&x->wait.lock); + timeout = action(timeout); + spin_lock_irq(&x->wait.lock); + } while (!x->done && timeout); + __remove_wait_queue(&x->wait, &wait); + if (!x->done) + return timeout; + } + x->done--; + return timeout ?: 1; +} + +static inline long __sched +__wait_for_common(struct completion *x, + long (*action)(long), long timeout, int state) +{ + might_sleep(); + + spin_lock_irq(&x->wait.lock); + timeout = do_wait_for_common(x, action, timeout, state); + spin_unlock_irq(&x->wait.lock); + return timeout; +} + +static long __sched +wait_for_common(struct completion *x, long timeout, int state) +{ + return __wait_for_common(x, schedule_timeout, timeout, state); +} + +static long __sched +wait_for_common_io(struct completion *x, long timeout, int state) +{ + return __wait_for_common(x, io_schedule_timeout, timeout, state); +} + +/** + * wait_for_completion: - waits for completion of a task + * @x: holds the state of this particular completion + * + * This waits to be signaled for completion of a specific task. It is NOT + * interruptible and there is no timeout. + * + * See also similar routines (i.e. wait_for_completion_timeout()) with timeout + * and interrupt capability. Also see complete(). + */ +void __sched wait_for_completion(struct completion *x) +{ + wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); +} +EXPORT_SYMBOL(wait_for_completion); + +/** + * wait_for_completion_timeout: - waits for completion of a task (w/timeout) + * @x: holds the state of this particular completion + * @timeout: timeout value in jiffies + * + * This waits for either a completion of a specific task to be signaled or for a + * specified timeout to expire. The timeout is in jiffies. It is not + * interruptible. + * + * Return: 0 if timed out, and positive (at least 1, or number of jiffies left + * till timeout) if completed. + */ +unsigned long __sched +wait_for_completion_timeout(struct completion *x, unsigned long timeout) +{ + return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE); +} +EXPORT_SYMBOL(wait_for_completion_timeout); + +/** + * wait_for_completion_io: - waits for completion of a task + * @x: holds the state of this particular completion + * + * This waits to be signaled for completion of a specific task. It is NOT + * interruptible and there is no timeout. The caller is accounted as waiting + * for IO. + */ +void __sched wait_for_completion_io(struct completion *x) +{ + wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); +} +EXPORT_SYMBOL(wait_for_completion_io); + +/** + * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout) + * @x: holds the state of this particular completion + * @timeout: timeout value in jiffies + * + * This waits for either a completion of a specific task to be signaled or for a + * specified timeout to expire. The timeout is in jiffies. It is not + * interruptible. The caller is accounted as waiting for IO. + * + * Return: 0 if timed out, and positive (at least 1, or number of jiffies left + * till timeout) if completed. + */ +unsigned long __sched +wait_for_completion_io_timeout(struct completion *x, unsigned long timeout) +{ + return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE); +} +EXPORT_SYMBOL(wait_for_completion_io_timeout); + +/** + * wait_for_completion_interruptible: - waits for completion of a task (w/intr) + * @x: holds the state of this particular completion + * + * This waits for completion of a specific task to be signaled. It is + * interruptible. + * + * Return: -ERESTARTSYS if interrupted, 0 if completed. + */ +int __sched wait_for_completion_interruptible(struct completion *x) +{ + long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE); + if (t == -ERESTARTSYS) + return t; + return 0; +} +EXPORT_SYMBOL(wait_for_completion_interruptible); + +/** + * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr)) + * @x: holds the state of this particular completion + * @timeout: timeout value in jiffies + * + * This waits for either a completion of a specific task to be signaled or for a + * specified timeout to expire. It is interruptible. The timeout is in jiffies. + * + * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1, + * or number of jiffies left till timeout) if completed. + */ +long __sched +wait_for_completion_interruptible_timeout(struct completion *x, + unsigned long timeout) +{ + return wait_for_common(x, timeout, TASK_INTERRUPTIBLE); +} +EXPORT_SYMBOL(wait_for_completion_interruptible_timeout); + +/** + * wait_for_completion_killable: - waits for completion of a task (killable) + * @x: holds the state of this particular completion + * + * This waits to be signaled for completion of a specific task. It can be + * interrupted by a kill signal. + * + * Return: -ERESTARTSYS if interrupted, 0 if completed. + */ +int __sched wait_for_completion_killable(struct completion *x) +{ + long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE); + if (t == -ERESTARTSYS) + return t; + return 0; +} +EXPORT_SYMBOL(wait_for_completion_killable); + +/** + * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable)) + * @x: holds the state of this particular completion + * @timeout: timeout value in jiffies + * + * This waits for either a completion of a specific task to be + * signaled or for a specified timeout to expire. It can be + * interrupted by a kill signal. The timeout is in jiffies. + * + * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1, + * or number of jiffies left till timeout) if completed. + */ +long __sched +wait_for_completion_killable_timeout(struct completion *x, + unsigned long timeout) +{ + return wait_for_common(x, timeout, TASK_KILLABLE); +} +EXPORT_SYMBOL(wait_for_completion_killable_timeout); + +/** + * try_wait_for_completion - try to decrement a completion without blocking + * @x: completion structure + * + * Return: 0 if a decrement cannot be done without blocking + * 1 if a decrement succeeded. + * + * If a completion is being used as a counting completion, + * attempt to decrement the counter without blocking. This + * enables us to avoid waiting if the resource the completion + * is protecting is not available. + */ +bool try_wait_for_completion(struct completion *x) +{ + unsigned long flags; + int ret = 1; + + spin_lock_irqsave(&x->wait.lock, flags); + if (!x->done) + ret = 0; + else + x->done--; + spin_unlock_irqrestore(&x->wait.lock, flags); + return ret; +} +EXPORT_SYMBOL(try_wait_for_completion); + +/** + * completion_done - Test to see if a completion has any waiters + * @x: completion structure + * + * Return: 0 if there are waiters (wait_for_completion() in progress) + * 1 if there are no waiters. + * + */ +bool completion_done(struct completion *x) +{ + unsigned long flags; + int ret = 1; + + spin_lock_irqsave(&x->wait.lock, flags); + if (!x->done) + ret = 0; + spin_unlock_irqrestore(&x->wait.lock, flags); + return ret; +} +EXPORT_SYMBOL(completion_done); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d9e35807eaeb3..c8659d8c0a0a5 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -73,7 +73,7 @@ #include #include #include -#include +#include #include #include @@ -82,93 +82,17 @@ #ifdef CONFIG_PARAVIRT #include #endif +#include #include "sched.h" #include "../workqueue_internal.h" #include "../smpboot.h" - +#ifdef CONFIG_MTPROF +#include "mt_sched_mon.h" +#endif #define CREATE_TRACE_POINTS #include -#include - -#define DLOG_SIZE 15000 -#define MAX_CTXSW_LATENCY 1000000000 -static DEFINE_PER_CPU(char[DLOG_SIZE], dbuf); -static DEFINE_PER_CPU(char *, dptr); - -#define dlog(x...) \ -do { \ - unsigned long dflags; \ - char *ptr, *buf; \ - int dcpu; \ - local_irq_save(dflags); \ - dcpu = smp_processor_id(); \ - buf = per_cpu(dbuf, dcpu); \ - ptr = per_cpu(dptr, dcpu); \ - ptr += snprintf(ptr, 10, "CPU %d ", dcpu); \ - ptr += snprintf(ptr, 490, x); \ - if (ptr - buf > DLOG_SIZE - 500) \ - ptr = buf; \ - per_cpu(dptr, dcpu) = ptr; \ - local_irq_restore(dflags); \ -} while (0) - -static atomic_t __su_instances; - -int su_instances(void) -{ - return atomic_read(&__su_instances); -} - -bool su_running(void) -{ - return su_instances() > 0; -} - -bool su_visible(void) -{ - kuid_t uid = current_uid(); - if (su_running()) - return true; - if (uid_eq(uid, GLOBAL_ROOT_UID) || uid_eq(uid, GLOBAL_SYSTEM_UID)) - return true; - return false; -} - -void su_exec(void) -{ - atomic_inc(&__su_instances); -} - -void su_exit(void) -{ - atomic_dec(&__su_instances); -} - -const char *task_event_names[] = {"PUT_PREV_TASK", "PICK_NEXT_TASK", - "TASK_WAKE", "TASK_MIGRATE", "TASK_UPDATE", - "IRQ_UPDATE"}; - -ATOMIC_NOTIFIER_HEAD(migration_notifier_head); -ATOMIC_NOTIFIER_HEAD(load_alert_notifier_head); - -#ifdef smp_mb__before_atomic -void __smp_mb__before_atomic(void) -{ - smp_mb__before_atomic(); -} -EXPORT_SYMBOL(__smp_mb__before_atomic); -#endif - -#ifdef smp_mb__after_atomic -void __smp_mb__after_atomic(void) -{ - smp_mb__after_atomic(); -} -EXPORT_SYMBOL(__smp_mb__after_atomic); -#endif - void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period) { unsigned long delta; @@ -202,6 +126,8 @@ void update_rq_clock(struct rq *rq) return; delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; + if (delta < 0) + return; rq->clock += delta; update_rq_clock_task(rq, delta); } @@ -259,12 +185,14 @@ struct static_key sched_feat_keys[__SCHED_FEAT_NR] = { static void sched_feat_disable(int i) { - static_key_disable(&sched_feat_keys[i]); + if (static_key_enabled(&sched_feat_keys[i])) + static_key_slow_dec(&sched_feat_keys[i]); } static void sched_feat_enable(int i) { - static_key_enable(&sched_feat_keys[i]); + if (!static_key_enabled(&sched_feat_keys[i])) + static_key_slow_inc(&sched_feat_keys[i]); } #else static void sched_feat_disable(int i) { }; @@ -304,6 +232,7 @@ sched_feat_write(struct file *filp, const char __user *ubuf, char buf[64]; char *cmp; int i; + struct inode *inode; if (cnt > 63) cnt = 63; @@ -314,7 +243,11 @@ sched_feat_write(struct file *filp, const char __user *ubuf, buf[cnt] = 0; cmp = strstrip(buf); + /* Ensure the static_key remains in a consistent state */ + inode = file_inode(filp); + mutex_lock(&inode->i_mutex); i = sched_feat_set(cmp); + mutex_unlock(&inode->i_mutex); if (i == __SCHED_FEAT_NR) return -EINVAL; @@ -387,9 +320,12 @@ static inline struct rq *__task_rq_lock(struct task_struct *p) for (;;) { rq = task_rq(p); raw_spin_lock(&rq->lock); - if (likely(rq == task_rq(p))) + if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) return rq; raw_spin_unlock(&rq->lock); + + while (unlikely(task_on_rq_migrating(p))) + cpu_relax(); } } @@ -406,10 +342,13 @@ static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags) raw_spin_lock_irqsave(&p->pi_lock, *flags); rq = task_rq(p); raw_spin_lock(&rq->lock); - if (likely(rq == task_rq(p))) + if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) return rq; raw_spin_unlock(&rq->lock); raw_spin_unlock_irqrestore(&p->pi_lock, *flags); + + while (unlikely(task_on_rq_migrating(p))) + cpu_relax(); } } @@ -446,13 +385,6 @@ static struct rq *this_rq_lock(void) #ifdef CONFIG_SCHED_HRTICK /* * Use HR-timers to deliver accurate preemption points. - * - * Its all a bit involved since we cannot program an hrt while holding the - * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a - * reschedule event. - * - * When we get rescheduled we reprogram the hrtick_timer outside of the - * rq->lock. */ static void hrtick_clear(struct rq *rq) @@ -480,22 +412,24 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer) } #ifdef CONFIG_SMP + +static int __hrtick_restart(struct rq *rq) +{ + struct hrtimer *timer = &rq->hrtick_timer; + ktime_t time = hrtimer_get_softexpires(timer); + + return __hrtimer_start_range_ns(timer, time, 0, HRTIMER_MODE_ABS_PINNED, 0); +} + /* * called from hardirq (IPI) context */ static void __hrtick_start(void *arg) { struct rq *rq = arg; - struct hrtimer *timer = &rq->hrtick_timer; - ktime_t soft, hard; - unsigned long delta; - - soft = hrtimer_get_softexpires(timer); - hard = hrtimer_get_expires(timer); - delta = ktime_to_ns(ktime_sub(hard, soft)); raw_spin_lock(&rq->lock); - __hrtimer_start_range_ns(timer, soft, delta, HRTIMER_MODE_ABS, 0); + __hrtick_restart(rq); rq->hrtick_csd_pending = 0; raw_spin_unlock(&rq->lock); } @@ -508,15 +442,22 @@ static void __hrtick_start(void *arg) void hrtick_start(struct rq *rq, u64 delay) { struct hrtimer *timer = &rq->hrtick_timer; - ktime_t time = ktime_add_ns(timer->base->get_time(), delay); + ktime_t time; + s64 delta; + + /* + * Don't schedule slices shorter than 10000ns, that just + * doesn't make sense and can cause timer DoS. + */ + delta = max_t(s64, delay, 10000LL); + time = ktime_add_ns(timer->base->get_time(), delta); hrtimer_set_expires(timer, time); if (rq == this_rq()) { - __hrtimer_start_range_ns(timer, ns_to_ktime(delay), 0, - HRTIMER_MODE_REL_PINNED, 0); + __hrtick_restart(rq); } else if (!rq->hrtick_csd_pending) { - __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0); + smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); rq->hrtick_csd_pending = 1; } } @@ -552,6 +493,11 @@ static __init void init_hrtick(void) */ void hrtick_start(struct rq *rq, u64 delay) { + /* + * Don't schedule slices shorter than 10000ns, that just + * doesn't make sense. Rely on vruntime for fairness. + */ + delay = max_t(u64, delay, 10000LL); __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0, HRTIMER_MODE_REL_PINNED, 0); } @@ -589,32 +535,99 @@ static inline void init_hrtick(void) #endif /* CONFIG_SCHED_HRTICK */ /* - * resched_task - mark a task 'to be rescheduled now'. + * cmpxchg based fetch_or, macro so it works for different integer types + */ +#define fetch_or(ptr, val) \ +({ typeof(*(ptr)) __old, __val = *(ptr); \ + for (;;) { \ + __old = cmpxchg((ptr), __val, __val | (val)); \ + if (__old == __val) \ + break; \ + __val = __old; \ + } \ + __old; \ +}) + +#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG) +/* + * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, + * this avoids any races wrt polling state changes and thereby avoids + * spurious IPIs. + */ +static bool set_nr_and_not_polling(struct task_struct *p) +{ + struct thread_info *ti = task_thread_info(p); + return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); +} + +/* + * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. + * + * If this returns true, then the idle task promises to call + * sched_ttwu_pending() and reschedule soon. + */ +static bool set_nr_if_polling(struct task_struct *p) +{ + struct thread_info *ti = task_thread_info(p); + typeof(ti->flags) old, val = ACCESS_ONCE(ti->flags); + + for (;;) { + if (!(val & _TIF_POLLING_NRFLAG)) + return false; + if (val & _TIF_NEED_RESCHED) + return true; + old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED); + if (old == val) + break; + val = old; + } + return true; +} + +#else +static bool set_nr_and_not_polling(struct task_struct *p) +{ + set_tsk_need_resched(p); + return true; +} + +#ifdef CONFIG_SMP +static bool set_nr_if_polling(struct task_struct *p) +{ + return false; +} +#endif +#endif + +/* + * resched_curr - mark rq's current task 'to be rescheduled now'. * * On UP this means the setting of the need_resched flag, on SMP it * might also involve a cross-CPU call to trigger the scheduler on * the target CPU. */ -#ifdef CONFIG_SMP -void resched_task(struct task_struct *p) +void resched_curr(struct rq *rq) { + struct task_struct *curr = rq->curr; int cpu; - assert_raw_spin_locked(&task_rq(p)->lock); + lockdep_assert_held(&rq->lock); - if (test_tsk_need_resched(p)) + if (test_tsk_need_resched(curr)) return; - set_tsk_need_resched(p); + cpu = cpu_of(rq); - cpu = task_cpu(p); - if (cpu == smp_processor_id()) + if (cpu == smp_processor_id()) { + set_tsk_need_resched(curr); + set_preempt_need_resched(); return; + } - /* NEED_RESCHED must be visible before we test polling */ - smp_mb(); - if (!tsk_is_polling(p)) + if (set_nr_and_not_polling(curr)) smp_send_reschedule(cpu); + else + trace_sched_wake_idle_without_ipi(cpu); } void resched_cpu(int cpu) @@ -624,10 +637,11 @@ void resched_cpu(int cpu) if (!raw_spin_trylock_irqsave(&rq->lock, flags)) return; - resched_task(cpu_curr(cpu)); + resched_curr(rq); raw_spin_unlock_irqrestore(&rq->lock, flags); } +#ifdef CONFIG_SMP #ifdef CONFIG_NO_HZ_COMMON /* * In the semi idle case, use the nearest busy cpu for migrating timers @@ -673,31 +687,20 @@ static void wake_up_idle_cpu(int cpu) if (cpu == smp_processor_id()) return; - /* - * This is safe, as this function is called with the timer - * wheel base lock of (cpu) held. When the CPU is on the way - * to idle and has not yet set rq->curr to idle then it will - * be serialized on the timer wheel base lock and take the new - * timer into account automatically. - */ - if (rq->curr != rq->idle) - return; - - /* - * We can set TIF_RESCHED on the idle task of the other CPU - * lockless. The worst case is that the other CPU runs the - * idle task through an additional NOOP schedule() - */ - set_tsk_need_resched(rq->idle); - - /* NEED_RESCHED must be visible before we test polling */ - smp_mb(); - if (!tsk_is_polling(rq->idle)) + if (set_nr_and_not_polling(rq->idle)) smp_send_reschedule(cpu); + else + trace_sched_wake_idle_without_ipi(cpu); } static bool wake_up_full_nohz_cpu(int cpu) { + /* + * We just need the target to call irq_exit() and re-evaluate + * the next tick. The nohz full kick at least implies that. + * If needed we can still optimize that later with an + * empty IRQ. + */ if (tick_nohz_full_cpu(cpu)) { if (cpu != smp_processor_id() || tick_nohz_tick_stopped()) @@ -744,18 +747,15 @@ static inline bool got_nohz_idle_kick(void) #ifdef CONFIG_NO_HZ_FULL bool sched_can_stop_tick(void) { - struct rq *rq; - - rq = this_rq(); - - /* Make sure rq->nr_running update is visible after the IPI */ - smp_rmb(); - - /* More than one running task need preemption */ - if (rq->nr_running > 1) - return false; + /* + * More than one running task need preemption. + * nr_running update is assumed to be visible + * after IPI is sent from wakers. + */ + if (this_rq()->nr_running > 1) + return false; - return true; + return true; } #endif /* CONFIG_NO_HZ_FULL */ @@ -763,7 +763,7 @@ void sched_avg_update(struct rq *rq) { s64 period = sched_avg_period(); - while ((s64)(rq->clock - rq->age_stamp) > period) { + while ((s64)(rq_clock(rq) - rq->age_stamp) > period) { /* * Inline assembly required to prevent the compiler * optimising this loop into a divmod call. @@ -775,30 +775,6 @@ void sched_avg_update(struct rq *rq) } } -/* - * Note C-state for (idle) cpus. - * - * @cstate = cstate index, 0 -> active state - * @wakeup_energy = energy spent in waking up cpu - * @wakeup_latency = latency to wakeup from cstate - * - */ -void -sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency) -{ - struct rq *rq = cpu_rq(cpu); - - rq->cstate = cstate; /* C1, C2 etc */ - rq->wakeup_energy = wakeup_energy; - rq->wakeup_latency = wakeup_latency; -} - -#else /* !CONFIG_SMP */ -void resched_task(struct task_struct *p) -{ - assert_raw_spin_locked(&task_rq(p)->lock); - set_tsk_need_resched(p); -} #endif /* CONFIG_SMP */ #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ @@ -867,17 +843,15 @@ static void set_load_weight(struct task_struct *p) static void enqueue_task(struct rq *rq, struct task_struct *p, int flags) { update_rq_clock(rq); - sched_info_queued(p); + sched_info_queued(rq, p); p->sched_class->enqueue_task(rq, p, flags); - trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_allowed)[0]); } static void dequeue_task(struct rq *rq, struct task_struct *p, int flags) { update_rq_clock(rq); - sched_info_dequeued(p); + sched_info_dequeued(rq, p); p->sched_class->dequeue_task(rq, p, flags); - trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_allowed)[0]); } void activate_task(struct rq *rq, struct task_struct *p, int flags) @@ -931,19 +905,13 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) #endif #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING if (static_key_false((¶virt_steal_rq_enabled))) { - u64 st; - steal = paravirt_steal_clock(cpu_of(rq)); steal -= rq->prev_steal_time_rq; if (unlikely(steal > delta)) steal = delta; - st = steal_ticks(steal); - steal = st * TICK_NSEC; - rq->prev_steal_time_rq += steal; - delta -= steal; } #endif @@ -951,7 +919,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) rq->clock_task += delta; #if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING) - if ((irq_delta + steal) && sched_feat(NONTASK_POWER)) + if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) sched_rt_avg_update(rq, irq_delta + steal); #endif } @@ -1005,7 +973,9 @@ static inline int normal_prio(struct task_struct *p) { int prio; - if (task_has_rt_policy(p)) + if (task_has_dl_policy(p)) + prio = MAX_DL_PRIO-1; + else if (task_has_rt_policy(p)) prio = MAX_RT_PRIO-1 - p->rt_priority; else prio = __normal_prio(p); @@ -1035,12 +1005,17 @@ static int effective_prio(struct task_struct *p) /** * task_curr - is this task currently executing on a CPU? * @p: the task in question. + * + * Return: 1 if the task is currently executing. 0 otherwise. */ inline int task_curr(const struct task_struct *p) { return cpu_curr(task_cpu(p)) == p; } +/* + * Can drop rq->lock because from sched_class::switched_from() methods drop it. + */ static inline void check_class_changed(struct rq *rq, struct task_struct *p, const struct sched_class *prev_class, int oldprio) @@ -1048,8 +1023,9 @@ static inline void check_class_changed(struct rq *rq, struct task_struct *p, if (prev_class != p->sched_class) { if (prev_class->switched_from) prev_class->switched_from(rq, p); + /* Possble rq->lock 'hole'. */ p->sched_class->switched_to(rq, p); - } else if (oldprio != p->prio) + } else if (oldprio != p->prio || dl_task(p)) p->sched_class->prio_changed(rq, p, oldprio); } @@ -1064,7 +1040,7 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) if (class == rq->curr->sched_class) break; if (class == p->sched_class) { - resched_task(rq->curr); + resched_curr(rq); break; } } @@ -1074,4426 +1050,2210 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) * A queue event has occurred, and we're going to schedule. In * this case, we can save a useless back to back clock update. */ - if (rq->curr->on_rq && test_tsk_need_resched(rq->curr)) + if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) rq->skip_clock_update = 1; } -static ATOMIC_NOTIFIER_HEAD(task_migration_notifier); +#ifdef CONFIG_SMP +/* + * This is how migration works: + * + * 1) we invoke migration_cpu_stop() on the target CPU using + * stop_one_cpu(). + * 2) stopper starts to run (implicitly forcing the migrated thread + * off the CPU) + * 3) it checks whether the migrated task is still in the wrong runqueue. + * 4) if it's in the wrong runqueue then the migration thread removes + * it and puts it into the right queue. + * 5) stopper completes and stop_one_cpu() returns and the migration + * is done. + */ -void register_task_migration_notifier(struct notifier_block *n) +/* + * move_queued_task - move a queued task to new rq. + * + * Returns (locked) new rq. Old rq's lock is released. + */ +static struct rq *move_queued_task(struct task_struct *p, int new_cpu) { - atomic_notifier_chain_register(&task_migration_notifier, n); -} + struct rq *rq = task_rq(p); -#ifdef CONFIG_SCHED_HMP + lockdep_assert_held(&rq->lock); -static int __init set_sched_enable_hmp(char *str) -{ - int enable_hmp = 0; + dequeue_task(rq, p, 0); + p->on_rq = TASK_ON_RQ_MIGRATING; + set_task_cpu(p, new_cpu); + raw_spin_unlock(&rq->lock); - get_option(&str, &enable_hmp); + rq = cpu_rq(new_cpu); - sched_enable_hmp = !!enable_hmp; + raw_spin_lock(&rq->lock); + BUG_ON(task_cpu(p) != new_cpu); + p->on_rq = TASK_ON_RQ_QUEUED; + enqueue_task(rq, p, 0); + check_preempt_curr(rq, p, 0); - return 0; + return rq; } -early_param("sched_enable_hmp", set_sched_enable_hmp); +struct migration_arg { + struct task_struct *task; + int dest_cpu; +}; -static int __init set_sched_enable_power_aware(char *str) +/* + * Move (not current) task off this cpu, onto dest cpu. We're doing + * this because either it can't run here any more (set_cpus_allowed() + * away from this CPU, or CPU going down), or because we're + * attempting to rebalance this task on exec (sched_exec). + * + * So we race with normal scheduler movements, but that's OK, as long + * as the task is no longer on this CPU. + * + * Returns non-zero if task was successfully migrated. + */ +static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) { - int enable_power_aware = 0; - - get_option(&str, &enable_power_aware); + struct rq *rq; + int ret = 0; - sysctl_sched_enable_power_aware = !!enable_power_aware; + if (unlikely(!cpu_active(dest_cpu))) + return ret; - return 0; -} + rq = cpu_rq(src_cpu); -early_param("sched_enable_power_aware", set_sched_enable_power_aware); + raw_spin_lock(&p->pi_lock); + raw_spin_lock(&rq->lock); + /* Already moved. */ + if (task_cpu(p) != src_cpu) + goto done; -static inline int got_boost_kick(void) -{ - int cpu = smp_processor_id(); - struct rq *rq = cpu_rq(cpu); + /* Affinity changed (again). */ + if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) + goto fail; - return test_bit(BOOST_KICK, &rq->hmp_flags); + /* + * If we're not on a rq, the next wake-up will ensure we're + * placed properly. + */ + if (task_on_rq_queued(p)) + rq = move_queued_task(p, dest_cpu); +done: + ret = 1; +fail: + raw_spin_unlock(&rq->lock); + raw_spin_unlock(&p->pi_lock); + return ret; } -static inline void clear_boost_kick(int cpu) +/* + * migration_cpu_stop - this will be executed by a highprio stopper thread + * and performs thread migration by bumping thread off CPU then + * 'pushing' onto another runqueue. + */ +static int migration_cpu_stop(void *data) { - struct rq *rq = cpu_rq(cpu); + struct migration_arg *arg = data; - clear_bit(BOOST_KICK, &rq->hmp_flags); + /* + * The original target cpu might have gone down and we might + * be on another cpu but it doesn't matter. + */ + local_irq_disable(); + /* + * We need to explicitly wake pending tasks before running + * __migrate_task() such that we will not miss enforcing cpus_allowed + * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. + */ + sched_ttwu_pending(); + __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu); + local_irq_enable(); + return 0; } -void boost_kick(int cpu) +void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) { - struct rq *rq = cpu_rq(cpu); + if (p->sched_class->set_cpus_allowed) + p->sched_class->set_cpus_allowed(p, new_mask); - if (!test_and_set_bit(BOOST_KICK, &rq->hmp_flags)) - smp_send_reschedule(cpu); + cpumask_copy(&p->cpus_allowed, new_mask); + p->nr_cpus_allowed = cpumask_weight(new_mask); } -/* Clear any HMP scheduler related requests pending from or on cpu */ -static inline void clear_hmp_request(int cpu) +/* + * Change a given task's CPU affinity. Migrate the thread to a + * proper CPU and schedule it away if the CPU it's executing on + * is removed from the allowed bitmask. + * + * NOTE: the caller must have a valid reference to the task, the + * task must not exit() & deallocate itself prematurely. The + * call is not atomic; no spinlocks may be held. + */ +int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) { - struct rq *rq = cpu_rq(cpu); unsigned long flags; + struct rq *rq; + unsigned int dest_cpu; + int ret = 0; - clear_boost_kick(cpu); - clear_reserved(cpu); - if (rq->push_task) { - raw_spin_lock_irqsave(&rq->lock, flags); - if (rq->push_task) { - clear_reserved(rq->push_cpu); - put_task_struct(rq->push_task); - rq->push_task = NULL; - } - rq->active_balance = 0; - raw_spin_unlock_irqrestore(&rq->lock, flags); - } -} + rq = task_rq_lock(p, &flags); -#else + if (cpumask_equal(&p->cpus_allowed, new_mask)) + goto out; -static inline int got_boost_kick(void) -{ - return 0; -} - -static inline void clear_boost_kick(int cpu) { } - -static inline void clear_hmp_request(int cpu) { } - -#endif /* CONFIG_SCHED_HMP */ - -#if defined(CONFIG_SCHED_HMP) - -/* - * sched_window_stats_policy, sched_account_wait_time, sched_ravg_hist_size, - * sched_migration_fixup, sched_freq_account_wait_time have a 'sysctl' copy - * associated with them. This is required for atomic update of those variables - * when being modifed via sysctl interface. - * - * IMPORTANT: Initialize both copies to same value!! - */ - -static __read_mostly unsigned int sched_ravg_hist_size = 5; -__read_mostly unsigned int sysctl_sched_ravg_hist_size = 5; - -static __read_mostly unsigned int sched_window_stats_policy = - WINDOW_STATS_MAX_RECENT_AVG; -__read_mostly unsigned int sysctl_sched_window_stats_policy = - WINDOW_STATS_MAX_RECENT_AVG; - -static __read_mostly unsigned int sched_account_wait_time = 1; -__read_mostly unsigned int sysctl_sched_account_wait_time = 1; - -__read_mostly unsigned int sysctl_sched_cpu_high_irqload = (10 * NSEC_PER_MSEC); - -#ifdef CONFIG_SCHED_FREQ_INPUT - -static __read_mostly unsigned int sched_migration_fixup = 1; -__read_mostly unsigned int sysctl_sched_migration_fixup = 1; - -static __read_mostly unsigned int sched_freq_account_wait_time; -__read_mostly unsigned int sysctl_sched_freq_account_wait_time; - -/* - * For increase, send notification if - * freq_required - cur_freq > sysctl_sched_freq_inc_notify - */ -__read_mostly int sysctl_sched_freq_inc_notify = 10 * 1024 * 1024; /* + 10GHz */ - -/* - * For decrease, send notification if - * cur_freq - freq_required > sysctl_sched_freq_dec_notify - */ -__read_mostly int sysctl_sched_freq_dec_notify = 10 * 1024 * 1024; /* - 10GHz */ - -static __read_mostly unsigned int sched_io_is_busy; - -#endif /* CONFIG_SCHED_FREQ_INPUT */ - -/* 1 -> use PELT based load stats, 0 -> use window-based load stats */ -unsigned int __read_mostly sched_use_pelt; - -unsigned int max_possible_efficiency = 1024; -unsigned int min_possible_efficiency = 1024; - -/* - * Maximum possible frequency across all cpus. Task demand and cpu - * capacity (cpu_power) metrics are scaled in reference to it. - */ -unsigned int max_possible_freq = 1; - -/* - * Minimum possible max_freq across all cpus. This will be same as - * max_possible_freq on homogeneous systems and could be different from - * max_possible_freq on heterogenous systems. min_max_freq is used to derive - * capacity (cpu_power) of cpus. - */ -unsigned int min_max_freq = 1; - -unsigned int max_capacity = 1024; /* max(rq->capacity) */ -unsigned int min_capacity = 1024; /* min(rq->capacity) */ -unsigned int max_load_scale_factor = 1024; /* max possible load scale factor */ -unsigned int max_possible_capacity = 1024; /* max(rq->max_possible_capacity) */ - -/* Mask of all CPUs that have max_possible_capacity */ -cpumask_t mpc_mask = CPU_MASK_ALL; - -/* Window size (in ns) */ -__read_mostly unsigned int sched_ravg_window = 10000000; - -/* Min window size (in ns) = 10ms */ -#define MIN_SCHED_RAVG_WINDOW 10000000 - -/* Max window size (in ns) = 1s */ -#define MAX_SCHED_RAVG_WINDOW 1000000000 - -/* Temporarily disable window-stats activity on all cpus */ -unsigned int __read_mostly sched_disable_window_stats; - -static unsigned int sync_cpu; - -#define EXITING_TASK_MARKER 0xdeaddead - -static inline int exiting_task(struct task_struct *p) -{ - return (p->ravg.sum_history[0] == EXITING_TASK_MARKER); -} - -static int __init set_sched_ravg_window(char *str) -{ - get_option(&str, &sched_ravg_window); - - sched_use_pelt = (sched_ravg_window < MIN_SCHED_RAVG_WINDOW || - sched_ravg_window > MAX_SCHED_RAVG_WINDOW); - - return 0; -} + if (!cpumask_intersects(new_mask, cpu_active_mask)) { + ret = -EINVAL; + goto out; + } -early_param("sched_ravg_window", set_sched_ravg_window); + do_set_cpus_allowed(p, new_mask); -static inline void -update_window_start(struct rq *rq, u64 wallclock) -{ - s64 delta; - int nr_windows; + /* Can the task run on the task's current CPU? If so, we're done */ + if (cpumask_test_cpu(task_cpu(p), new_mask)) + goto out; - delta = wallclock - rq->window_start; - BUG_ON(delta < 0); - if (delta < sched_ravg_window) - return; + dest_cpu = cpumask_any_and(cpu_active_mask, new_mask); + if (task_running(rq, p) || p->state == TASK_WAKING) { + struct migration_arg arg = { p, dest_cpu }; + /* Need help from migration thread: drop lock and wait. */ + task_rq_unlock(rq, p, &flags); + stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); + tlb_migrate_finish(p->mm); + return 0; + } else if (task_on_rq_queued(p)) + rq = move_queued_task(p, dest_cpu); +out: + task_rq_unlock(rq, p, &flags); - nr_windows = div64_u64(delta, sched_ravg_window); - rq->window_start += (u64)nr_windows * (u64)sched_ravg_window; + return ret; } +EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); -static inline u64 scale_exec_time(u64 delta, struct rq *rq) +void set_task_cpu(struct task_struct *p, unsigned int new_cpu) { - unsigned int cur_freq = rq->cur_freq; - int sf; - - if (unlikely(cur_freq > max_possible_freq || - (cur_freq == rq->max_freq && - rq->max_freq < rq->max_possible_freq))) - cur_freq = rq->max_possible_freq; - - /* round up div64 */ - delta = div64_u64(delta * cur_freq + max_possible_freq - 1, - max_possible_freq); - - sf = DIV_ROUND_UP(rq->efficiency * 1024, max_possible_efficiency); - - delta *= sf; - delta >>= 10; +#ifdef CONFIG_SCHED_DEBUG + /* + * We should never call set_task_cpu() on a blocked task, + * ttwu() will sort out the placement. + */ + WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING && + !(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE)); - return delta; -} +#ifdef CONFIG_LOCKDEP + /* + * The caller should hold either p->pi_lock or rq->lock, when changing + * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. + * + * sched_move_task() holds both and thus holding either pins the cgroup, + * see task_group(). + * + * Furthermore, all task_rq users should acquire both locks, see + * task_rq_lock(). + */ + WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || + lockdep_is_held(&task_rq(p)->lock))); +#endif +#endif -#ifdef CONFIG_SCHED_FREQ_INPUT + trace_sched_migrate_task(p, new_cpu, 0); -static inline int cpu_is_waiting_on_io(struct rq *rq) -{ - if (!sched_io_is_busy) - return 0; + if (task_cpu(p) != new_cpu) { + if (p->sched_class->migrate_task_rq) + p->sched_class->migrate_task_rq(p, new_cpu); + p->se.nr_migrations++; + perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0); + } - return atomic_read(&rq->nr_iowait); + __set_task_cpu(p, new_cpu); } -/* Does freq_required sufficiently exceed or fall behind cur_freq? */ -static inline int -nearly_same_freq(unsigned int cur_freq, unsigned int freq_required) +#ifdef TJK_HMP +static void __migrate_swap_task(struct task_struct *p, int cpu) { - int delta = freq_required - cur_freq; + if (task_on_rq_queued(p)) { + struct rq *src_rq, *dst_rq; - if (freq_required > cur_freq) - return delta < sysctl_sched_freq_inc_notify; + src_rq = task_rq(p); + dst_rq = cpu_rq(cpu); - delta = -delta; - - return delta < sysctl_sched_freq_dec_notify; + deactivate_task(src_rq, p, 0); + set_task_cpu(p, cpu); + activate_task(dst_rq, p, 0); + check_preempt_curr(dst_rq, p, 0); + } else { + /* + * Task isn't running anymore; make it appear like we migrated + * it before it went to sleep. This means on wakeup we make the + * previous cpu our targer instead of where it really is. + */ + p->wake_cpu = cpu; + } } +#endif -/* Convert busy time to frequency equivalent */ -static inline unsigned int load_to_freq(struct rq *rq, u64 load) -{ - unsigned int freq; - - load = scale_load_to_cpu(load, cpu_of(rq)); - load *= 128; - load = div64_u64(load, max_task_load()); - - freq = load * rq->max_possible_freq; - freq /= 128; - - return freq; -} +struct migration_swap_arg { + struct task_struct *src_task, *dst_task; + int src_cpu, dst_cpu; +}; -/* Should scheduler alert governor for changing frequency? */ -static int send_notification(struct rq *rq) +#ifdef TJK_HMP +static int migrate_swap_stop(void *data) { - unsigned int cur_freq, freq_required; - unsigned long flags; - int rc = 0; + struct migration_swap_arg *arg = data; + struct rq *src_rq, *dst_rq; + int ret = -EAGAIN; - if (!sched_enable_hmp) - return 0; + src_rq = cpu_rq(arg->src_cpu); + dst_rq = cpu_rq(arg->dst_cpu); - cur_freq = load_to_freq(rq, rq->old_busy_time); - freq_required = load_to_freq(rq, rq->prev_runnable_sum); + double_raw_lock(&arg->src_task->pi_lock, + &arg->dst_task->pi_lock); + double_rq_lock(src_rq, dst_rq); + if (task_cpu(arg->dst_task) != arg->dst_cpu) + goto unlock; - if (nearly_same_freq(cur_freq, freq_required)) - return 0; + if (task_cpu(arg->src_task) != arg->src_cpu) + goto unlock; - raw_spin_lock_irqsave(&rq->lock, flags); - if (!rq->notifier_sent) { - rq->notifier_sent = 1; - rc = 1; - } - raw_spin_unlock_irqrestore(&rq->lock, flags); + if (!cpumask_test_cpu(arg->dst_cpu, tsk_cpus_allowed(arg->src_task))) + goto unlock; - return rc; -} + if (!cpumask_test_cpu(arg->src_cpu, tsk_cpus_allowed(arg->dst_task))) + goto unlock; -/* Alert governor if there is a need to change frequency */ -void check_for_freq_change(struct rq *rq) -{ - int cpu = cpu_of(rq); + __migrate_swap_task(arg->src_task, arg->dst_cpu); + __migrate_swap_task(arg->dst_task, arg->src_cpu); - if (!send_notification(rq)) - return; + ret = 0; - trace_sched_freq_alert(cpu, rq->old_busy_time, rq->prev_runnable_sum); +unlock: + double_rq_unlock(src_rq, dst_rq); + raw_spin_unlock(&arg->dst_task->pi_lock); + raw_spin_unlock(&arg->src_task->pi_lock); - atomic_notifier_call_chain( - &load_alert_notifier_head, 0, - (void *)(long)cpu); + return ret; } +#endif -static int account_busy_for_cpu_time(struct rq *rq, struct task_struct *p, - u64 irqtime, int event) +/* + * Cross migrate two tasks + */ +int migrate_swap(struct task_struct *cur, struct task_struct *p) { - if (is_idle_task(p)) { - /* TASK_WAKE && TASK_MIGRATE is not possible on idle task! */ - if (event == PICK_NEXT_TASK) - return 0; - - /* PUT_PREV_TASK, TASK_UPDATE && IRQ_UPDATE are left */ - return irqtime || cpu_is_waiting_on_io(rq); - } + struct migration_swap_arg arg; + int ret = -EINVAL; - if (event == TASK_WAKE) - return 0; + arg = (struct migration_swap_arg){ + .src_task = cur, + .src_cpu = task_cpu(cur), + .dst_task = p, + .dst_cpu = task_cpu(p), + }; - if (event == PUT_PREV_TASK || event == IRQ_UPDATE || - event == TASK_UPDATE) - return 1; + if (arg.src_cpu == arg.dst_cpu) + goto out; - /* Only TASK_MIGRATE && PICK_NEXT_TASK left */ - return sched_freq_account_wait_time; -} + /* + * These three tests are all lockless; this is OK since all of them + * will be re-checked with proper locks held further down the line. + */ + if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu)) + goto out; -static inline int -heavy_task_wakeup(struct task_struct *p, struct rq *rq, int event) -{ - u32 task_demand = p->ravg.demand; + if (!cpumask_test_cpu(arg.dst_cpu, tsk_cpus_allowed(arg.src_task))) + goto out; - if (!sched_heavy_task || event != TASK_WAKE || - task_demand < sched_heavy_task || exiting_task(p)) - return 0; + if (!cpumask_test_cpu(arg.src_cpu, tsk_cpus_allowed(arg.dst_task))) + goto out; - if (p->ravg.mark_start > rq->window_start) - return 0; +#if 0 + trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); + ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); +#else + ret = -EINVAL; +#endif - /* has a full window elapsed since task slept? */ - return (rq->window_start - p->ravg.mark_start > sched_ravg_window); +out: + return ret; } /* - * Account cpu activity in its busy time counters (rq->curr/prev_runnable_sum) + * wait_task_inactive - wait for a thread to unschedule. + * + * If @match_state is nonzero, it's the @p->state value just checked and + * not expected to change. If it changes, i.e. @p might have woken up, + * then return zero. When we succeed in waiting for @p to be off its CPU, + * we return a positive number (its total switch count). If a second call + * a short while later returns the same number, the caller can be sure that + * @p has remained unscheduled the whole time. + * + * The caller must ensure that the task *will* unschedule sometime soon, + * else this function might spin for a *long* time. This function can't + * be called with interrupts off, or it may introduce deadlock with + * smp_call_function() if an IPI is sent by the same process we are + * waiting to become inactive. */ -static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, - int event, u64 wallclock, u64 irqtime) +unsigned long wait_task_inactive(struct task_struct *p, long match_state) { - int new_window, nr_full_windows = 0; - int p_is_curr_task = (p == rq->curr); - u64 mark_start = p->ravg.mark_start; - u64 window_start = rq->window_start; - u32 window_size = sched_ravg_window; - u64 delta; + unsigned long flags; + int running, queued; + unsigned long ncsw; + struct rq *rq; - new_window = mark_start < window_start; - if (new_window) - nr_full_windows = div64_u64((window_start - mark_start), - window_size); + for (;;) { + /* + * We do the initial early heuristics without holding + * any task-queue locks at all. We'll only try to get + * the runqueue lock when things look like they will + * work out! + */ + rq = task_rq(p); - /* Handle per-task window rollover. We don't care about the idle - * task or exiting tasks. */ - if (new_window && !is_idle_task(p) && !exiting_task(p)) { - u32 curr_window = 0; + /* + * If the task is actively running on another CPU + * still, just relax and busy-wait without holding + * any locks. + * + * NOTE! Since we don't hold any locks, it's not + * even sure that "rq" stays as the right runqueue! + * But we don't care, since "task_running()" will + * return false if the runqueue has changed and p + * is actually now running somewhere else! + */ + while (task_running(rq, p)) { + if (match_state && unlikely(p->state != match_state)) + return 0; + cpu_relax(); + } - if (!nr_full_windows) - curr_window = p->ravg.curr_window; + /* + * Ok, time to look more closely! We need the rq + * lock now, to be *sure*. If we're wrong, we'll + * just go back and repeat. + */ + rq = task_rq_lock(p, &flags); + trace_sched_wait_task(p); + running = task_running(rq, p); + queued = task_on_rq_queued(p); + ncsw = 0; + if (!match_state || p->state == match_state) + ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ + task_rq_unlock(rq, p, &flags); - p->ravg.prev_window = curr_window; - p->ravg.curr_window = 0; - } + /* + * If it changed from the expected state, bail out now. + */ + if (unlikely(!ncsw)) + break; - if (!account_busy_for_cpu_time(rq, p, irqtime, event)) { - /* account_busy_for_cpu_time() = 0, so no update to the - * task's current window needs to be made. This could be - * for example + /* + * Was it really running after all now that we + * checked with the proper locks actually held? * - * - a wakeup event on a task within the current - * window (!new_window below, no action required), - * - switching to a new task from idle (PICK_NEXT_TASK) - * in a new window where irqtime is 0 and we aren't - * waiting on IO */ + * Oops. Go back and try again.. + */ + if (unlikely(running)) { + cpu_relax(); + continue; + } - if (!new_window) - return; + /* + * It's not enough that it's not actively running, + * it must be off the runqueue _entirely_, and not + * preempted! + * + * So if it was still runnable (but just not actively + * running right now), it's preempted, and we should + * yield - it could be a while. + */ + if (unlikely(queued)) { + ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ); - /* A new window has started. The RQ demand must be rolled - * over if p is the current task. */ - if (p_is_curr_task) { - u64 prev_sum = 0; - - /* p is either idle task or an exiting task */ - if (!nr_full_windows) - prev_sum = rq->curr_runnable_sum; - - rq->prev_runnable_sum = prev_sum; - rq->curr_runnable_sum = 0; - - } else if (heavy_task_wakeup(p, rq, event)) { - /* A new window has started. If p is a waking - * heavy task its prev_window contribution is faked - * to be its window-based demand. Note that this can - * introduce phantom load into the system depending - * on the window policy and task behavior. This feature - * can be controlled via the sched_heavy_task - * tunable. */ - p->ravg.prev_window = p->ravg.demand; - rq->prev_runnable_sum += p->ravg.demand; + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_hrtimeout(&to, HRTIMER_MODE_REL); + continue; } - return; + /* + * Ahh, all good. It wasn't running, and it wasn't + * runnable, which means that it will never become + * running in the future either. We're all done! + */ + break; } - if (!new_window) { - /* account_busy_for_cpu_time() = 1 so busy time needs - * to be accounted to the current window. No rollover - * since we didn't start a new window. An example of this is - * when a task starts execution and then sleeps within the - * same window. */ + return ncsw; +} - if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq)) - delta = wallclock - mark_start; - else - delta = irqtime; - delta = scale_exec_time(delta, rq); - rq->curr_runnable_sum += delta; - if (!is_idle_task(p) && !exiting_task(p)) - p->ravg.curr_window += delta; +/*** + * kick_process - kick a running thread to enter/exit the kernel + * @p: the to-be-kicked thread + * + * Cause a process which is running on another CPU to enter + * kernel-mode, without any delay. (to get signals handled.) + * + * NOTE: this function doesn't have to take the runqueue lock, + * because all it wants to ensure is that the remote task enters + * the kernel. If the IPI races and the task has been migrated + * to another CPU then no harm is done and the purpose has been + * achieved as well. + */ +void kick_process(struct task_struct *p) +{ + int cpu; - return; - } + preempt_disable(); + cpu = task_cpu(p); + if ((cpu != smp_processor_id()) && task_curr(p)) + smp_send_reschedule(cpu); + preempt_enable(); +} +EXPORT_SYMBOL_GPL(kick_process); - if (!p_is_curr_task) { - /* account_busy_for_cpu_time() = 1 so busy time needs - * to be accounted to the current window. A new window - * has also started, but p is not the current task, so the - * window is not rolled over - just split up and account - * as necessary into curr and prev. The window is only - * rolled over when a new window is processed for the current - * task. - * - * Irqtime can't be accounted by a task that isn't the - * currently running task. */ - - if (!nr_full_windows) { - /* A full window hasn't elapsed, account partial - * contribution to previous completed window. */ - delta = scale_exec_time(window_start - mark_start, rq); - if (!exiting_task(p)) - p->ravg.prev_window += delta; - } else { - /* Since at least one full window has elapsed, - * the contribution to the previous window is the - * full window (window_size). */ - delta = scale_exec_time(window_size, rq); - if (!exiting_task(p)) - p->ravg.prev_window = delta; - } - rq->prev_runnable_sum += delta; +/* + * ->cpus_allowed is protected by both rq->lock and p->pi_lock + */ +static int select_fallback_rq(int cpu, struct task_struct *p) +{ + int nid = cpu_to_node(cpu); + const struct cpumask *nodemask = NULL; + enum { cpuset, possible, fail } state = cpuset; + int dest_cpu; - /* Account piece of busy time in the current window. */ - delta = scale_exec_time(wallclock - window_start, rq); - rq->curr_runnable_sum += delta; - if (!exiting_task(p)) - p->ravg.curr_window = delta; + /* + * If the node that the cpu is on has been offlined, cpu_to_node() + * will return -1. There is no cpu on the node, and we should + * select the cpu on the other node. + */ + if (nid != -1) { + nodemask = cpumask_of_node(nid); - return; + /* Look for allowed, online CPU in same node. */ + for_each_cpu(dest_cpu, nodemask) { + if (!cpu_online(dest_cpu)) + continue; + if (!cpu_active(dest_cpu)) + continue; + if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) + return dest_cpu; + } } - if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq)) { - /* account_busy_for_cpu_time() = 1 so busy time needs - * to be accounted to the current window. A new window - * has started and p is the current task so rollover is - * needed. If any of these three above conditions are true - * then this busy time can't be accounted as irqtime. - * - * Busy time for the idle task or exiting tasks need not - * be accounted. - * - * An example of this would be a task that starts execution - * and then sleeps once a new window has begun. */ - - if (!nr_full_windows) { - /* A full window hasn't elapsed, account partial - * contribution to previous completed window. */ - delta = scale_exec_time(window_start - mark_start, rq); - if (!is_idle_task(p) && !exiting_task(p)) - p->ravg.prev_window += delta; - delta += rq->curr_runnable_sum; - } else { - /* Since at least one full window has elapsed, - * the contribution to the previous window is the - * full window (window_size). */ - delta = scale_exec_time(window_size, rq); - if (!is_idle_task(p) && !exiting_task(p)) - p->ravg.prev_window = delta; + for (;;) { + /* Any allowed, online CPU? */ + for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) { + if (!cpu_online(dest_cpu)) + continue; + if (!cpu_active(dest_cpu)) + continue; + goto out; } - /* Rollover is done here by overwriting the values in - * prev_runnable_sum and curr_runnable_sum. */ - rq->prev_runnable_sum = delta; - /* Account piece of busy time in the current window. */ - delta = scale_exec_time(wallclock - window_start, rq); - rq->curr_runnable_sum = delta; - if (!is_idle_task(p) && !exiting_task(p)) - p->ravg.curr_window = delta; + switch (state) { + case cpuset: + /* No more Mr. Nice Guy. */ + cpuset_cpus_allowed_fallback(p); + state = possible; + break; + + case possible: + do_set_cpus_allowed(p, cpu_possible_mask); + state = fail; + break; - return; + case fail: + BUG(); + break; + } } - if (irqtime) { - /* account_busy_for_cpu_time() = 1 so busy time needs - * to be accounted to the current window. A new window - * has started and p is the current task so rollover is - * needed. The current task must be the idle task because - * irqtime is not accounted for any other task. - * - * Irqtime will be accounted each time we process IRQ activity - * after a period of idleness, so we know the IRQ busy time - * started at wallclock - irqtime. */ - - BUG_ON(!is_idle_task(p)); - mark_start = wallclock - irqtime; - - /* Roll window over. If IRQ busy time was just in the current - * window then that is all that need be accounted. */ - rq->prev_runnable_sum = rq->curr_runnable_sum; - if (mark_start > window_start) { - rq->curr_runnable_sum = scale_exec_time(irqtime, rq); - return; +out: + if (state != cpuset) { + /* + * Don't tell them about moving exiting tasks or + * kernel threads (both mm NULL), since they never + * leave kernel. + */ + if (p->mm && printk_ratelimit()) { + printk_deferred("process %d (%s) no longer affine to cpu%d\n", + task_pid_nr(p), p->comm, cpu); } + } - /* The IRQ busy time spanned multiple windows. Process the - * busy time preceding the current window start first. */ - delta = window_start - mark_start; - if (delta > window_size) - delta = window_size; - delta = scale_exec_time(delta, rq); - rq->prev_runnable_sum += delta; + return dest_cpu; +} - /* Process the remaining IRQ busy time in the current window. */ - delta = wallclock - window_start; - rq->curr_runnable_sum = scale_exec_time(delta, rq); +/* + * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable. + */ +static inline +int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) +{ + cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags); - return; - } + /* + * In order not to call set_task_cpu() on a blocking task we need + * to rely on ttwu() to place the task on a valid ->cpus_allowed + * cpu. + * + * Since this is common to all placement strategies, this lives here. + * + * [ this allows ->select_task() to simply return task_cpu(p) and + * not worry about this generic constraint ] + */ + if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) || + !cpu_online(cpu))) + cpu = select_fallback_rq(task_cpu(p), p); - BUG(); + return cpu; } -#else /* CONFIG_SCHED_FREQ_INPUT */ -static inline void update_cpu_busy_time(struct task_struct *p, struct rq *rq, - int event, u64 wallclock, u64 irqtime) +static void update_avg(u64 *avg, u64 sample) { + s64 diff = sample - *avg; + *avg += diff >> 3; } +#endif /* CONFIG_SMP */ -#endif /* CONFIG_SCHED_FREQ_INPUT */ - -static int account_busy_for_task_demand(struct task_struct *p, int event) +static void +ttwu_stat(struct task_struct *p, int cpu, int wake_flags) { - /* No need to bother updating task demand for exiting tasks - * or the idle task. */ - if (exiting_task(p) || is_idle_task(p)) - return 0; +#ifdef CONFIG_SCHEDSTATS + struct rq *rq = this_rq(); - /* When a task is waking up it is completing a segment of non-busy - * time. Likewise, if wait time is not treated as busy time, then - * when a task begins to run or is migrated, it is not running and - * is completing a segment of non-busy time. */ - if (event == TASK_WAKE || (!sched_account_wait_time && - (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) - return 0; +#ifdef CONFIG_SMP + int this_cpu = smp_processor_id(); - return 1; -} + if (cpu == this_cpu) { + schedstat_inc(rq, ttwu_local); + schedstat_inc(p, se.statistics.nr_wakeups_local); + } else { + struct sched_domain *sd; -/* - * Called when new window is starting for a task, to record cpu usage over - * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 - * when, say, a real-time task runs without preemption for several windows at a - * stretch. - */ -static void update_history(struct rq *rq, struct task_struct *p, - u32 runtime, int samples, int event) -{ - u32 *hist = &p->ravg.sum_history[0]; - int ridx, widx; - u32 max = 0, avg, demand; - u64 sum = 0; - - /* Ignore windows where task had no activity */ - if (!runtime || is_idle_task(p) || exiting_task(p) || !samples) - goto done; - -#ifdef VENDOR_EDIT - if (p->ravg.mitigated) { - /* update history accroding to previous records - * e.g. 60% loading in big cluster becomes 60% in smallest one - */ - for (ridx = 0; ridx < sched_ravg_hist_size; ridx++) { - hist[ridx] = div64_u64((u64)hist[ridx] * 1024 - , (u64)max_load_scale_factor); - sum += hist[ridx]; - if (hist[ridx] > max) - max = hist[ridx]; + schedstat_inc(p, se.statistics.nr_wakeups_remote); + rcu_read_lock(); + for_each_domain(this_cpu, sd) { + if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { + schedstat_inc(sd, ttwu_wake_remote); + break; + } } - goto update_demand; - } -#endif - /* Push new 'runtime' value onto stack */ - widx = sched_ravg_hist_size - 1; - ridx = widx - samples; - for (; ridx >= 0; --widx, --ridx) { - hist[widx] = hist[ridx]; - sum += hist[widx]; - if (hist[widx] > max) - max = hist[widx]; + rcu_read_unlock(); } - for (widx = 0; widx < samples && widx < sched_ravg_hist_size; widx++) { - hist[widx] = runtime; - sum += hist[widx]; - if (hist[widx] > max) - max = hist[widx]; - } + if (wake_flags & WF_MIGRATED) + schedstat_inc(p, se.statistics.nr_wakeups_migrate); -#ifdef VENDOR_EDIT -update_demand: -#endif - p->ravg.sum = 0; - if (p->on_rq) - p->sched_class->dec_hmp_sched_stats(rq, p); - - avg = div64_u64(sum, sched_ravg_hist_size); - - if (sched_window_stats_policy == WINDOW_STATS_RECENT) - demand = runtime; - else if (sched_window_stats_policy == WINDOW_STATS_MAX) - demand = max; - else if (sched_window_stats_policy == WINDOW_STATS_AVG) - demand = avg; - else - demand = max(avg, runtime); +#endif /* CONFIG_SMP */ - p->ravg.demand = demand; + schedstat_inc(rq, ttwu_count); + schedstat_inc(p, se.statistics.nr_wakeups); - if (p->on_rq) - p->sched_class->inc_hmp_sched_stats(rq, p); + if (wake_flags & WF_SYNC) + schedstat_inc(p, se.statistics.nr_wakeups_sync); -done: - trace_sched_update_history(rq, p, runtime, samples, event); +#endif /* CONFIG_SCHEDSTATS */ } -static void add_to_task_demand(struct rq *rq, struct task_struct *p, - u64 delta) +static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags) { - delta = scale_exec_time(delta, rq); - p->ravg.sum += delta; - if (unlikely(p->ravg.sum > sched_ravg_window)) - p->ravg.sum = sched_ravg_window; + activate_task(rq, p, en_flags); + p->on_rq = TASK_ON_RQ_QUEUED; + + /* if a worker is waking up, notify workqueue */ + if (p->flags & PF_WQ_WORKER) + wq_worker_waking_up(p, cpu_of(rq)); } /* - * Account cpu demand of task and/or update task's cpu demand history - * - * ms = p->ravg.mark_start; - * wc = wallclock - * ws = rq->window_start - * - * Three possibilities: - * - * a) Task event is contained within one window. - * window_start < mark_start < wallclock - * - * ws ms wc - * | | | - * V V V - * |---------------| - * - * In this case, p->ravg.sum is updated *iff* event is appropriate - * (ex: event == PUT_PREV_TASK) - * - * b) Task event spans two windows. - * mark_start < window_start < wallclock - * - * ms ws wc - * | | | - * V V V - * -----|------------------- - * - * In this case, p->ravg.sum is updated with (ws - ms) *iff* event - * is appropriate, then a new window sample is recorded followed - * by p->ravg.sum being set to (wc - ws) *iff* event is appropriate. - * - * c) Task event spans more than two windows. - * - * ms ws_tmp ws wc - * | | | | - * V V V V - * ---|-------|-------|-------|-------|------ - * | | - * |<------ nr_full_windows ------>| - * - * In this case, p->ravg.sum is updated with (ws_tmp - ms) first *iff* - * event is appropriate, window sample of p->ravg.sum is recorded, - * 'nr_full_window' samples of window_size is also recorded *iff* - * event is appropriate and finally p->ravg.sum is set to (wc - ws) - * *iff* event is appropriate. - * - * IMPORTANT : Leave p->ravg.mark_start unchanged, as update_cpu_busy_time() - * depends on it! + * Mark the task runnable and perform wakeup-preemption. */ -static void update_task_demand(struct task_struct *p, struct rq *rq, - int event, u64 wallclock) +static void +ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) { - u64 mark_start = p->ravg.mark_start; - u64 delta, window_start = rq->window_start; - int new_window, nr_full_windows; - u32 window_size = sched_ravg_window; - - new_window = mark_start < window_start; -#ifdef VENDOR_EDIT - if (!account_busy_for_task_demand(p, event) || p->ravg.mitigated) { -#else - if (!account_busy_for_task_demand(p, event)) { -#endif - if (new_window) - /* If the time accounted isn't being accounted as - * busy time, and a new window started, only the - * previous window need be closed out with the - * pre-existing demand. Multiple windows may have - * elapsed, but since empty windows are dropped, - * it is not necessary to account those. */ - update_history(rq, p, p->ravg.sum, 1, event); -#ifdef VENDOR_EDIT - if (p->ravg.mitigated) { - /* force update history */ - update_history(rq, p, 1, RAVG_HIST_SIZE_MAX, event); - p->ravg.mitigated = 0; - } -#endif - return; - } - - if (!new_window) { - /* The simple case - busy time contained within the existing - * window. */ - add_to_task_demand(rq, p, wallclock - mark_start); - return; - } + check_preempt_curr(rq, p, wake_flags); + trace_sched_wakeup(p, true); - /* Busy time spans at least two windows. Temporarily rewind - * window_start to first window boundary after mark_start. */ - delta = window_start - mark_start; - nr_full_windows = div64_u64(delta, window_size); - window_start -= (u64)nr_full_windows * (u64)window_size; + p->state = TASK_RUNNING; +#ifdef CONFIG_SMP + if (p->sched_class->task_woken) + p->sched_class->task_woken(rq, p); - /* Process (window_start - mark_start) first */ - add_to_task_demand(rq, p, window_start - mark_start); + if (rq->idle_stamp) { + u64 delta = rq_clock(rq) - rq->idle_stamp; + u64 max = 2*rq->max_idle_balance_cost; - /* Push new sample(s) into task's demand history */ - update_history(rq, p, p->ravg.sum, 1, event); - if (nr_full_windows) - update_history(rq, p, scale_exec_time(window_size, rq), - nr_full_windows, event); + update_avg(&rq->avg_idle, delta); - /* Roll window_start back to current to process any remainder - * in current window. */ - window_start += (u64)nr_full_windows * (u64)window_size; + if (rq->avg_idle > max) + rq->avg_idle = max; - /* Process (wallclock - window_start) next */ - mark_start = window_start; - add_to_task_demand(rq, p, wallclock - mark_start); + rq->idle_stamp = 0; + } +#endif } -/* Reflect task activity on its demand and cpu's busy time statistics */ -static void update_task_ravg(struct task_struct *p, struct rq *rq, - int event, u64 wallclock, u64 irqtime) +static void +ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags) { - if (sched_use_pelt || !rq->window_start || sched_disable_window_stats) - return; - - lockdep_assert_held(&rq->lock); - - update_window_start(rq, wallclock); +#ifdef CONFIG_SMP + if (p->sched_contributes_to_load) + rq->nr_uninterruptible--; +#endif - if (!p->ravg.mark_start) - goto done; + ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING); + ttwu_do_wakeup(rq, p, wake_flags); +} - update_task_demand(p, rq, event, wallclock); - update_cpu_busy_time(p, rq, event, wallclock, irqtime); +/* + * Called in case the task @p isn't fully descheduled from its runqueue, + * in this case we must do a remote wakeup. Its a 'light' wakeup though, + * since all we need to do is flip p->state to TASK_RUNNING, since + * the task is still ->on_rq. + */ +static int ttwu_remote(struct task_struct *p, int wake_flags) +{ + struct rq *rq; + int ret = 0; -done: - trace_sched_update_task_ravg(p, rq, event, wallclock, irqtime); + rq = __task_rq_lock(p); + if (task_on_rq_queued(p)) { + /* check_preempt_curr() may use rq clock */ + update_rq_clock(rq); + ttwu_do_wakeup(rq, p, wake_flags); + ret = 1; + } + __task_rq_unlock(rq); - p->ravg.mark_start = wallclock; + return ret; } -void sched_account_irqtime(int cpu, struct task_struct *curr, - u64 delta, u64 wallclock) +#ifdef CONFIG_SMP +void sched_ttwu_pending(void) { - struct rq *rq = cpu_rq(cpu); - unsigned long flags, nr_windows; - u64 cur_jiffies_ts, now; - - raw_spin_lock_irqsave(&rq->lock, flags); - - now = sched_clock(); - delta += (now - wallclock); - cur_jiffies_ts = get_jiffies_64(); + struct rq *rq = this_rq(); + struct llist_node *llist = llist_del_all(&rq->wake_list); + struct task_struct *p; + unsigned long flags; - if (is_idle_task(curr)) - update_task_ravg(curr, rq, IRQ_UPDATE, now, delta); + if (!llist) + return; - nr_windows = cur_jiffies_ts - rq->irqload_ts; + raw_spin_lock_irqsave(&rq->lock, flags); - if (nr_windows) { - if (nr_windows < 10) { - /* Decay CPU's irqload by 3/4 for each window. */ - rq->avg_irqload *= (3 * nr_windows); - rq->avg_irqload = div64_u64(rq->avg_irqload, - 4 * nr_windows); - } else { - rq->avg_irqload = 0; - } - rq->avg_irqload += rq->cur_irqload; - rq->cur_irqload = 0; + while (llist) { + p = llist_entry(llist, struct task_struct, wake_entry); + llist = llist_next(llist); + ttwu_do_activate(rq, p, 0); } - rq->cur_irqload += delta; - rq->irqload_ts = cur_jiffies_ts; raw_spin_unlock_irqrestore(&rq->lock, flags); } -unsigned long __weak arch_get_cpu_efficiency(int cpu) -{ - return SCHED_LOAD_SCALE; -} +enum ipi_msg_type { + IPI_RESCHEDULE, + IPI_CALL_FUNC, + IPI_CALL_FUNC_SINGLE, + IPI_CPU_STOP, + IPI_TIMER, + IPI_IRQ_WORK, +}; -static void init_cpu_efficiency(void) +void scheduler_ipi(void) { - int i, efficiency; - unsigned int max = 0, min = UINT_MAX; + /* + * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting + * TIF_NEED_RESCHED remotely (for the first time) will also send + * this IPI. + */ + preempt_fold_need_resched(); - if (!sched_enable_hmp) + if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick()) { +#ifdef CONFIG_MTPROF + mt_trace_ISR_start(IPI_RESCHEDULE); + mt_trace_ISR_end(IPI_RESCHEDULE); +#endif return; - - for_each_possible_cpu(i) { - efficiency = arch_get_cpu_efficiency(i); - cpu_rq(i)->efficiency = efficiency; - - if (efficiency > max) - max = efficiency; - if (efficiency < min) - min = efficiency; } - BUG_ON(!max || !min); - - max_possible_efficiency = max; - min_possible_efficiency = min; -} - -static void reset_task_stats(struct task_struct *p) -{ - u32 sum = 0; - - if (exiting_task(p)) - sum = EXITING_TASK_MARKER; + /* + * Not all reschedule IPI handlers call irq_enter/irq_exit, since + * traditionally all their work was done from the interrupt return + * path. Now that we actually do some work, we need to make sure + * we do call them. + * + * Some archs already do call them, luckily irq_enter/exit nest + * properly. + * + * Arguably we should visit all archs and update all handlers, + * however a fair share of IPIs are still resched only so this would + * somewhat pessimize the simple resched case. + */ + irq_enter(); +#ifdef CONFIG_MTPROF + mt_trace_ISR_start(IPI_RESCHEDULE); +#endif + sched_ttwu_pending(); - memset(&p->ravg, 0, sizeof(struct ravg)); - /* Retain EXITING_TASK marker */ - p->ravg.sum_history[0] = sum; + /* + * Check if someone kicked us for doing the nohz idle load balance. + */ + if (unlikely(got_nohz_idle_kick())) { + this_rq()->idle_balance = 1; + raise_softirq_irqoff(SCHED_SOFTIRQ); + } +#ifdef CONFIG_MTPROF + mt_trace_ISR_end(IPI_RESCHEDULE); +#endif + irq_exit(); } -static inline void mark_task_starting(struct task_struct *p) +static void ttwu_queue_remote(struct task_struct *p, int cpu) { - struct rq *rq = task_rq(p); - u64 wallclock = sched_clock(); + struct rq *rq = cpu_rq(cpu); - if (!rq->window_start || sched_disable_window_stats) { - reset_task_stats(p); - return; + if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) { + if (!set_nr_if_polling(rq->idle)) + smp_send_reschedule(cpu); + else + trace_sched_wake_idle_without_ipi(cpu); } - - p->ravg.mark_start = wallclock; } -static inline void set_window_start(struct rq *rq) +void wake_up_if_idle(int cpu) { - int cpu = cpu_of(rq); - struct rq *sync_rq = cpu_rq(sync_cpu); + struct rq *rq = cpu_rq(cpu); + unsigned long flags; - if (rq->window_start || !sched_enable_hmp || - !sched_clock_initialized() || !sched_clock_cpu(cpu)) - return; + rcu_read_lock(); - if (cpu == sync_cpu) { - rq->window_start = sched_clock(); + if (!is_idle_task(rcu_dereference(rq->curr))) + goto out; + + if (set_nr_if_polling(rq->idle)) { + trace_sched_wake_idle_without_ipi(cpu); } else { - raw_spin_unlock(&rq->lock); - double_rq_lock(rq, sync_rq); - rq->window_start = cpu_rq(sync_cpu)->window_start; -#ifdef CONFIG_SCHED_FREQ_INPUT - rq->curr_runnable_sum = rq->prev_runnable_sum = 0; -#endif - raw_spin_unlock(&sync_rq->lock); + raw_spin_lock_irqsave(&rq->lock, flags); + if (is_idle_task(rq->curr)) + smp_send_reschedule(cpu); + /* Else cpu is not in idle, do nothing here */ + raw_spin_unlock_irqrestore(&rq->lock, flags); } - rq->curr->ravg.mark_start = rq->window_start; +out: + rcu_read_unlock(); } -static inline void migrate_sync_cpu(int cpu) +bool cpus_share_cache(int this_cpu, int that_cpu) { - if (cpu == sync_cpu) - sync_cpu = smp_processor_id(); + return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); } +#endif /* CONFIG_SMP */ -static void reset_all_task_stats(void) +static void ttwu_queue(struct task_struct *p, int cpu) { - struct task_struct *g, *p; + struct rq *rq = cpu_rq(cpu); - read_lock(&tasklist_lock); - do_each_thread(g, p) { - reset_task_stats(p); - } while_each_thread(g, p); - read_unlock(&tasklist_lock); +#if defined(CONFIG_SMP) + if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { + sched_clock_cpu(cpu); /* sync clocks x-cpu */ + ttwu_queue_remote(p, cpu); + return; + } +#endif + + raw_spin_lock(&rq->lock); + ttwu_do_activate(rq, p, 0); + raw_spin_unlock(&rq->lock); } -/* - * sched_exit() - Set EXITING_TASK_MARKER in task's ravg.demand field +/** + * try_to_wake_up - wake up a thread + * @p: the thread to be awakened + * @state: the mask of task states that can be woken + * @wake_flags: wake modifier flags (WF_*) * - * Stop accounting (exiting) task's future cpu usage + * Put it on the run-queue if it's not already there. The "current" + * thread is always on the run-queue (except when the actual + * re-schedule is in progress), and as such you're allowed to do + * the simpler "current->state = TASK_RUNNING" to mark yourself + * runnable without the overhead of this. * - * We need this so that reset_all_windows_stats() can function correctly. - * reset_all_window_stats() depends on do_each_thread/for_each_thread task - * iterators to reset *all* task's statistics. Exiting tasks however become - * invisible to those iterators. sched_exit() is called on a exiting task prior - * to being removed from task_list, which will let reset_all_window_stats() - * function correctly. + * Return: %true if @p was woken up, %false if it was already running. + * or @state didn't match @p's state. */ -void sched_exit(struct task_struct *p) +static int +try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) { unsigned long flags; - int cpu = get_cpu(); - struct rq *rq = cpu_rq(cpu); - u64 wallclock; + int cpu, success = 0; - raw_spin_lock_irqsave(&rq->lock, flags); - /* rq->curr == p */ - wallclock = sched_clock(); - update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); - dequeue_task(rq, p, 0); - reset_task_stats(p); - p->ravg.mark_start = wallclock; - p->ravg.sum_history[0] = EXITING_TASK_MARKER; - enqueue_task(rq, p, 0); - raw_spin_unlock_irqrestore(&rq->lock, flags); + /* + * If we are going to wake up a thread waiting for CONDITION we + * need to ensure that CONDITION=1 done by the caller can not be + * reordered with p->state check below. This pairs with mb() in + * set_current_state() the waiting thread does. + */ + smp_mb__before_spinlock(); + raw_spin_lock_irqsave(&p->pi_lock, flags); + if (!(p->state & state)) + goto out; - put_cpu(); -} + success = 1; /* we're going to change ->state */ + cpu = task_cpu(p); -static void disable_window_stats(void) -{ - unsigned long flags; - int i; + if (p->on_rq && ttwu_remote(p, wake_flags)) + goto stat; - local_irq_save(flags); - for_each_possible_cpu(i) - raw_spin_lock(&cpu_rq(i)->lock); +#ifdef CONFIG_SMP + /* + * If the owning (remote) cpu is still in the middle of schedule() with + * this task as prev, wait until its done referencing the task. + */ + while (p->on_cpu) + cpu_relax(); + /* + * Pairs with the smp_wmb() in finish_lock_switch(). + */ + smp_rmb(); - sched_disable_window_stats = 1; + p->sched_contributes_to_load = !!task_contributes_to_load(p); + p->state = TASK_WAKING; - for_each_possible_cpu(i) - raw_spin_unlock(&cpu_rq(i)->lock); + if (p->sched_class->task_waking) + p->sched_class->task_waking(p); - local_irq_restore(flags); -} + cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags); + if (task_cpu(p) != cpu) { + wake_flags |= WF_MIGRATED; + set_task_cpu(p, cpu); + } +#endif /* CONFIG_SMP */ -/* Called with all cpu's rq->lock held */ -static void enable_window_stats(void) -{ - sched_disable_window_stats = 0; + ttwu_queue(p, cpu); +stat: + ttwu_stat(p, cpu, wake_flags); +out: + raw_spin_unlock_irqrestore(&p->pi_lock, flags); + return success; } -enum reset_reason_code { - WINDOW_CHANGE, - POLICY_CHANGE, - ACCOUNT_WAIT_TIME_CHANGE, - HIST_SIZE_CHANGE, - MIGRATION_FIXUP_CHANGE, - FREQ_ACCOUNT_WAIT_TIME_CHANGE -}; - -const char *sched_window_reset_reasons[] = { - "WINDOW_CHANGE", - "POLICY_CHANGE", - "ACCOUNT_WAIT_TIME_CHANGE", - "HIST_SIZE_CHANGE", - "MIGRATION_FIXUP_CHANGE", - "FREQ_ACCOUNT_WAIT_TIME_CHANGE"}; - -/* Called with IRQs enabled */ -void reset_all_window_stats(u64 window_start, unsigned int window_size) +/** + * try_to_wake_up_local - try to wake up a local task with rq lock held + * @p: the thread to be awakened + * + * Put @p on the run-queue if it's not already there. The caller must + * ensure that this_rq() is locked, @p is bound to this_rq() and not + * the current task. + */ +static void try_to_wake_up_local(struct task_struct *p) { - int cpu; - unsigned long flags; - u64 start_ts = sched_clock(); - int reason = WINDOW_CHANGE; - unsigned int old = 0, new = 0; - unsigned int old_window_size = sched_ravg_window; - - disable_window_stats(); + struct rq *rq = task_rq(p); - reset_all_task_stats(); + if (WARN_ON_ONCE(rq != this_rq()) || + WARN_ON_ONCE(p == current)) + return; - local_irq_save(flags); + lockdep_assert_held(&rq->lock); - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); + if (!raw_spin_trylock(&p->pi_lock)) { + raw_spin_unlock(&rq->lock); + raw_spin_lock(&p->pi_lock); raw_spin_lock(&rq->lock); } - if (window_size) { - sched_ravg_window = window_size * TICK_NSEC; - set_hmp_defaults(); - } + if (!(p->state & TASK_NORMAL)) + goto out; - enable_window_stats(); + if (!task_on_rq_queued(p)) + ttwu_activate(rq, p, ENQUEUE_WAKEUP); - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); + ttwu_do_wakeup(rq, p, 0); + ttwu_stat(p, smp_processor_id(), 0); +out: + raw_spin_unlock(&p->pi_lock); +} - if (window_start) { - u32 mostly_idle_load = rq->mostly_idle_load; +/** + * wake_up_process - Wake up a specific process + * @p: The process to be woken up. + * + * Attempt to wake up the nominated process and move it to the set of runnable + * processes. + * + * Return: 1 if the process was woken up, 0 if it was already running. + * + * It may be assumed that this function implies a write memory barrier before + * changing the task state if and only if any tasks are woken up. + */ +int wake_up_process(struct task_struct *p) +{ + WARN_ON(task_is_stopped_or_traced(p)); + return try_to_wake_up(p, TASK_NORMAL, 0); +} +EXPORT_SYMBOL(wake_up_process); - rq->window_start = window_start; - rq->mostly_idle_load = div64_u64((u64)mostly_idle_load * - (u64)sched_ravg_window, (u64)old_window_size); - } -#ifdef CONFIG_SCHED_FREQ_INPUT - rq->curr_runnable_sum = rq->prev_runnable_sum = 0; -#endif - reset_cpu_hmp_stats(cpu, 1); - - fixup_nr_big_small_task(cpu, 0); - } - - if (sched_window_stats_policy != sysctl_sched_window_stats_policy) { - reason = POLICY_CHANGE; - old = sched_window_stats_policy; - new = sysctl_sched_window_stats_policy; - sched_window_stats_policy = sysctl_sched_window_stats_policy; - } else if (sched_account_wait_time != sysctl_sched_account_wait_time) { - reason = ACCOUNT_WAIT_TIME_CHANGE; - old = sched_account_wait_time; - new = sysctl_sched_account_wait_time; - sched_account_wait_time = sysctl_sched_account_wait_time; - } else if (sched_ravg_hist_size != sysctl_sched_ravg_hist_size) { - reason = HIST_SIZE_CHANGE; - old = sched_ravg_hist_size; - new = sysctl_sched_ravg_hist_size; - sched_ravg_hist_size = sysctl_sched_ravg_hist_size; - } -#ifdef CONFIG_SCHED_FREQ_INPUT - else if (sched_migration_fixup != sysctl_sched_migration_fixup) { - reason = MIGRATION_FIXUP_CHANGE; - old = sched_migration_fixup; - new = sysctl_sched_migration_fixup; - sched_migration_fixup = sysctl_sched_migration_fixup; - } else if (sched_freq_account_wait_time != - sysctl_sched_freq_account_wait_time) { - reason = FREQ_ACCOUNT_WAIT_TIME_CHANGE; - old = sched_freq_account_wait_time; - new = sysctl_sched_freq_account_wait_time; - sched_freq_account_wait_time = - sysctl_sched_freq_account_wait_time; - } -#endif +int wake_up_state(struct task_struct *p, unsigned int state) +{ + return try_to_wake_up(p, state, 0); +} - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - raw_spin_unlock(&rq->lock); - } +/* + * This function clears the sched_dl_entity static params. + */ +void __dl_clear_params(struct task_struct *p) +{ + struct sched_dl_entity *dl_se = &p->dl; - local_irq_restore(flags); + dl_se->dl_runtime = 0; + dl_se->dl_deadline = 0; + dl_se->dl_period = 0; + dl_se->flags = 0; + dl_se->dl_bw = 0; - trace_sched_reset_all_window_stats(window_start, window_size, - sched_clock() - start_ts, reason, old, new); + dl_se->dl_throttled = 0; + dl_se->dl_new = 1; + dl_se->dl_yielded = 0; } -#ifdef CONFIG_SCHED_FREQ_INPUT - -static inline u64 -scale_load_to_freq(u64 load, unsigned int src_freq, unsigned int dst_freq) +/* + * Perform scheduler related setup for a newly forked process p. + * p is forked by current. + * + * __sched_fork() is basic setup used by init_idle() too: + */ +static void __sched_fork(unsigned long clone_flags, struct task_struct *p) { - return div64_u64(load * (u64)src_freq, (u64)dst_freq); -} + p->on_rq = 0; -void sched_get_cpus_busy(unsigned long *busy, const struct cpumask *query_cpus) -{ - unsigned long flags; - struct rq *rq; - const int cpus = cpumask_weight(query_cpus); - u64 load[cpus]; - unsigned int cur_freq[cpus], max_freq[cpus]; - int notifier_sent[cpus]; - int cpu, i = 0; - unsigned int window_size; - - if (unlikely(cpus == 0)) - return; + p->se.on_rq = 0; + p->se.exec_start = 0; + p->se.sum_exec_runtime = 0; + p->se.prev_sum_exec_runtime = 0; + p->se.nr_migrations = 0; + p->se.vruntime = 0; +#ifdef CONFIG_SMP + p->se.avg.decay_count = 0; +#endif + INIT_LIST_HEAD(&p->se.group_node); - /* - * This function could be called in timer context, and the - * current task may have been executing for a long time. Ensure - * that the window stats are current by doing an update. - */ - local_irq_save(flags); - for_each_cpu(cpu, query_cpus) - raw_spin_lock(&cpu_rq(cpu)->lock); +#ifdef CONFIG_SCHEDSTATS + memset(&p->se.statistics, 0, sizeof(p->se.statistics)); +#endif - window_size = sched_ravg_window; + RB_CLEAR_NODE(&p->dl.rb_node); + init_dl_task_timer(&p->dl); + __dl_clear_params(p); - for_each_cpu(cpu, query_cpus) { - rq = cpu_rq(cpu); + INIT_LIST_HEAD(&p->rt.run_list); - update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), 0); - load[i] = rq->old_busy_time = rq->prev_runnable_sum; - /* - * Scale load in reference to rq->max_possible_freq. - * - * Note that scale_load_to_cpu() scales load in reference to - * rq->max_freq. - */ - load[i] = scale_load_to_cpu(load[i], cpu); +#ifdef CONFIG_PREEMPT_NOTIFIERS + INIT_HLIST_HEAD(&p->preempt_notifiers); +#endif - notifier_sent[i] = rq->notifier_sent; - rq->notifier_sent = 0; - cur_freq[i] = rq->cur_freq; - max_freq[i] = rq->max_freq; - i++; +#ifdef CONFIG_NUMA_BALANCING + if (p->mm && atomic_read(&p->mm->mm_users) == 1) { + p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); + p->mm->numa_scan_seq = 0; } - for_each_cpu(cpu, query_cpus) - raw_spin_unlock(&(cpu_rq(cpu))->lock); - local_irq_restore(flags); - - i = 0; - for_each_cpu(cpu, query_cpus) { - rq = cpu_rq(cpu); + if (clone_flags & CLONE_VM) + p->numa_preferred_nid = current->numa_preferred_nid; + else + p->numa_preferred_nid = -1; - if (!notifier_sent[i]) { - load[i] = scale_load_to_freq(load[i], max_freq[i], - cur_freq[i]); - if (load[i] > window_size) - load[i] = window_size; - load[i] = scale_load_to_freq(load[i], cur_freq[i], - rq->max_possible_freq); - } else { - load[i] = scale_load_to_freq(load[i], max_freq[i], - rq->max_possible_freq); - } - - busy[i] = div64_u64(load[i], NSEC_PER_USEC); + p->node_stamp = 0ULL; + p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0; + p->numa_scan_period = sysctl_numa_balancing_scan_delay; + p->numa_work.next = &p->numa_work; + p->numa_faults_memory = NULL; + p->numa_faults_buffer_memory = NULL; + p->last_task_numa_placement = 0; + p->last_sum_exec_runtime = 0; - trace_sched_get_busy(cpu, busy[i]); - i++; - } + INIT_LIST_HEAD(&p->numa_entry); + p->numa_group = NULL; +#endif /* CONFIG_NUMA_BALANCING */ } -unsigned long sched_get_busy(int cpu) +#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_SCHED_DEBUG +void set_numabalancing_state(bool enabled) { - struct cpumask query_cpu = CPU_MASK_NONE; - unsigned long busy; - - cpumask_set_cpu(cpu, &query_cpu); - sched_get_cpus_busy(&busy, &query_cpu); - - return busy; + if (enabled) + sched_feat_set("NUMA"); + else + sched_feat_set("NO_NUMA"); } +#else +__read_mostly bool numabalancing_enabled; -void sched_set_io_is_busy(int val) +void set_numabalancing_state(bool enabled) { - sched_io_is_busy = val; + numabalancing_enabled = enabled; } +#endif /* CONFIG_SCHED_DEBUG */ -int sched_set_window(u64 window_start, unsigned int window_size) +#ifdef CONFIG_PROC_SYSCTL +int sysctl_numa_balancing(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) { - u64 now, cur_jiffies, jiffy_sched_clock; - s64 ws; - unsigned long flags; - - if (sched_use_pelt || - (window_size * TICK_NSEC < MIN_SCHED_RAVG_WINDOW)) - return -EINVAL; - - mutex_lock(&policy_mutex); - - /* Get a consistent view of sched_clock, jiffies, and the time - * since the last jiffy (based on last_jiffies_update). */ - local_irq_save(flags); - cur_jiffies = jiffy_to_sched_clock(&now, &jiffy_sched_clock); - local_irq_restore(flags); - - /* translate window_start from jiffies to nanoseconds */ - ws = (window_start - cur_jiffies); /* jiffy difference */ - ws *= TICK_NSEC; - ws += jiffy_sched_clock; - - /* roll back calculated window start so that it is in - * the past (window stats must have a current window) */ - while (ws > now) - ws -= (window_size * TICK_NSEC); - - BUG_ON(sched_clock() < ws); - - reset_all_window_stats(ws, window_size); + struct ctl_table t; + int err; + int state = numabalancing_enabled; - mutex_unlock(&policy_mutex); + if (write && !capable(CAP_SYS_ADMIN)) + return -EPERM; - return 0; + t = *table; + t.data = &state; + err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); + if (err < 0) + return err; + if (write) + set_numabalancing_state(state); + return err; } +#endif +#endif -static void fixup_busy_time(struct task_struct *p, int new_cpu) +/* + * fork()/clone()-time setup: + */ +int sched_fork(unsigned long clone_flags, struct task_struct *p) { - struct rq *src_rq = task_rq(p); - struct rq *dest_rq = cpu_rq(new_cpu); - u64 wallclock; - - if (!sched_enable_hmp || !sched_migration_fixup || - exiting_task(p) || (!p->on_rq && p->state != TASK_WAKING)) - return; - - if (p->state == TASK_WAKING) - double_rq_lock(src_rq, dest_rq); - - if (sched_disable_window_stats) - goto done; - - wallclock = sched_clock(); - - update_task_ravg(task_rq(p)->curr, task_rq(p), - TASK_UPDATE, - wallclock, 0); - update_task_ravg(dest_rq->curr, dest_rq, - TASK_UPDATE, wallclock, 0); + unsigned long flags; + int cpu = get_cpu(); + __sched_fork(clone_flags, p); /* - * In case of migration of task on runqueue, on_rq =1, - * however its load is removed from its runqueue. - * update_task_ravg() below can update its demand, which - * will require its load on runqueue to be adjusted to - * reflect new demand. Restore load temporarily for such - * task on its runqueue + * We mark the process as running here. This guarantees that + * nobody will actually run it, and a signal or other external + * event cannot wake it up and insert it on the runqueue either. */ - if (p->on_rq) - p->sched_class->inc_hmp_sched_stats(src_rq, p); + p->state = TASK_RUNNING; - update_task_ravg(p, task_rq(p), TASK_MIGRATE, - wallclock, 0); + /* + * Make sure we do not leak PI boosting priority to the child. + */ + p->prio = current->normal_prio; /* - * Remove task's load from rq as its now migrating to - * another cpu. + * Revert to default priority/policy on fork if requested. */ - if (p->on_rq) - p->sched_class->dec_hmp_sched_stats(src_rq, p); + if (unlikely(p->sched_reset_on_fork)) { + if (task_has_dl_policy(p) || task_has_rt_policy(p)) { + p->policy = SCHED_NORMAL; + p->static_prio = NICE_TO_PRIO(0); + p->rt_priority = 0; + } else if (PRIO_TO_NICE(p->static_prio) < 0) + p->static_prio = NICE_TO_PRIO(0); - if (p->ravg.curr_window) { - src_rq->curr_runnable_sum -= p->ravg.curr_window; - dest_rq->curr_runnable_sum += p->ravg.curr_window; - } + p->prio = p->normal_prio = __normal_prio(p); + set_load_weight(p); - if (p->ravg.prev_window) { - src_rq->prev_runnable_sum -= p->ravg.prev_window; - dest_rq->prev_runnable_sum += p->ravg.prev_window; + /* + * We don't need the reset flag anymore after the fork. It has + * fulfilled its duty: + */ + p->sched_reset_on_fork = 0; } - BUG_ON((s64)src_rq->prev_runnable_sum < 0); - BUG_ON((s64)src_rq->curr_runnable_sum < 0); - - trace_sched_migration_update_sum(src_rq, p); - trace_sched_migration_update_sum(dest_rq, p); + if (dl_prio(p->prio)) { + put_cpu(); + return -EAGAIN; + } else if (rt_prio(p->prio)) { + p->sched_class = &rt_sched_class; + } else { + p->sched_class = &fair_sched_class; + } -done: - if (p->state == TASK_WAKING) - double_rq_unlock(src_rq, dest_rq); -} + if (p->sched_class->task_fork) + p->sched_class->task_fork(p); -#else + /* + * The child is not yet in the pid-hash so no cgroup attach races, + * and the cgroup is pinned to this child due to cgroup_fork() + * is ran before sched_fork(). + * + * Silence PROVE_RCU. + */ + raw_spin_lock_irqsave(&p->pi_lock, flags); + set_task_cpu(p, cpu); + raw_spin_unlock_irqrestore(&p->pi_lock, flags); -static inline void fixup_busy_time(struct task_struct *p, int new_cpu) { } +#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) + if (likely(sched_info_on())) + memset(&p->sched_info, 0, sizeof(p->sched_info)); +#endif +#if defined(CONFIG_SMP) + p->on_cpu = 0; +#endif + task_thread_info(p)->preempt_count = 1; +#ifdef CONFIG_SMP + plist_node_init(&p->pushable_tasks, MAX_PRIO); + RB_CLEAR_NODE(&p->pushable_dl_tasks); +#endif -static inline int -heavy_task_wakeup(struct task_struct *p, struct rq *rq, int event) -{ + put_cpu(); return 0; } -#endif /* CONFIG_SCHED_FREQ_INPUT */ - -/* Keep track of max/min capacity possible across CPUs "currently" */ -static void __update_min_max_capacity(void) +unsigned long to_ratio(u64 period, u64 runtime) { - int i; - int max = 0, min = INT_MAX; + if (runtime == RUNTIME_INF) + return 1ULL << 20; - for_each_online_cpu(i) { - if (cpu_rq(i)->capacity > max) - max = cpu_rq(i)->capacity; - if (cpu_rq(i)->capacity < min) - min = cpu_rq(i)->capacity; - } + /* + * Doing this here saves a lot of checks in all + * the calling paths, and returning zero seems + * safe for them anyway. + */ + if (period == 0) + return 0; - max_capacity = max; - min_capacity = min; + return div64_u64(runtime << 20, period); } -static void update_min_max_capacity(void) +#ifdef CONFIG_SMP +inline struct dl_bw *dl_bw_of(int i) { - unsigned long flags; - int i; - - local_irq_save(flags); - for_each_possible_cpu(i) - raw_spin_lock(&cpu_rq(i)->lock); - - __update_min_max_capacity(); - - for_each_possible_cpu(i) - raw_spin_unlock(&cpu_rq(i)->lock); - local_irq_restore(flags); + rcu_lockdep_assert(rcu_read_lock_sched_held(), + "sched RCU must be held"); + return &cpu_rq(i)->rd->dl_bw; } -/* - * Return 'capacity' of a cpu in reference to "least" efficient cpu, such that - * least efficient cpu gets capacity of 1024 - */ -unsigned long capacity_scale_cpu_efficiency(int cpu) +static inline int dl_bw_cpus(int i) { - return (1024 * cpu_rq(cpu)->efficiency) / min_possible_efficiency; -} + struct root_domain *rd = cpu_rq(i)->rd; + int cpus = 0; -/* - * Return 'capacity' of a cpu in reference to cpu with lowest max_freq - * (min_max_freq), such that one with lowest max_freq gets capacity of 1024. - */ -unsigned long capacity_scale_cpu_freq(int cpu) + rcu_lockdep_assert(rcu_read_lock_sched_held(), + "sched RCU must be held"); + for_each_cpu_and(i, rd->span, cpu_active_mask) + cpus++; + + return cpus; +} +#else +inline struct dl_bw *dl_bw_of(int i) { - return (1024 * cpu_rq(cpu)->max_freq) / min_max_freq; + return &cpu_rq(i)->dl.dl_bw; } -/* - * Return load_scale_factor of a cpu in reference to "most" efficient cpu, so - * that "most" efficient cpu gets a load_scale_factor of 1 - */ -static inline unsigned long load_scale_cpu_efficiency(int cpu) +static inline int dl_bw_cpus(int i) { - return DIV_ROUND_UP(1024 * max_possible_efficiency, - cpu_rq(cpu)->efficiency); + return 1; } +#endif -/* - * Return load_scale_factor of a cpu in reference to cpu with best max_freq - * (max_possible_freq), so that one with best max_freq gets a load_scale_factor - * of 1. - */ -static inline unsigned long load_scale_cpu_freq(int cpu) +static inline +void __dl_clear(struct dl_bw *dl_b, u64 tsk_bw) { - return DIV_ROUND_UP(1024 * max_possible_freq, cpu_rq(cpu)->max_freq); + dl_b->total_bw -= tsk_bw; } -static int compute_capacity(int cpu) +static inline +void __dl_add(struct dl_bw *dl_b, u64 tsk_bw) { - int capacity = 1024; - - capacity *= capacity_scale_cpu_efficiency(cpu); - capacity >>= 10; - - capacity *= capacity_scale_cpu_freq(cpu); - capacity >>= 10; - - return capacity; + dl_b->total_bw += tsk_bw; } -static int compute_load_scale_factor(int cpu) +static inline +bool __dl_overflow(struct dl_bw *dl_b, int cpus, u64 old_bw, u64 new_bw) { - int load_scale = 1024; - - /* - * load_scale_factor accounts for the fact that task load - * is in reference to "best" performing cpu. Task's load will need to be - * scaled (up) by a factor to determine suitability to be placed on a - * (little) cpu. - */ - load_scale *= load_scale_cpu_efficiency(cpu); - load_scale >>= 10; - - load_scale *= load_scale_cpu_freq(cpu); - load_scale >>= 10; - - return load_scale; + return dl_b->bw != -1 && + dl_b->bw * cpus < dl_b->total_bw - old_bw + new_bw; } -static int cpufreq_notifier_policy(struct notifier_block *nb, - unsigned long val, void *data) +/* + * We must be sure that accepting a new task (or allowing changing the + * parameters of an existing one) is consistent with the bandwidth + * constraints. If yes, this function also accordingly updates the currently + * allocated bandwidth to reflect the new situation. + * + * This function is called while holding p's rq->lock. + * + * XXX we should delay bw change until the task's 0-lag point, see + * __setparam_dl(). + */ +static int dl_overflow(struct task_struct *p, int policy, + const struct sched_attr *attr) { - struct cpufreq_policy *policy = (struct cpufreq_policy *)data; - int i, update_max = 0; - u64 highest_mpc = 0, highest_mplsf = 0; - const struct cpumask *cpus = policy->related_cpus; - unsigned int orig_min_max_freq = min_max_freq; - unsigned int orig_max_possible_freq = max_possible_freq; - /* Initialized to policy->max in case policy->related_cpus is empty! */ - unsigned int orig_max_freq = policy->max; - if (val != CPUFREQ_NOTIFY && val != CPUFREQ_REMOVE_POLICY) - return 0; + struct dl_bw *dl_b = dl_bw_of(task_cpu(p)); + u64 period = attr->sched_period ?: attr->sched_deadline; + u64 runtime = attr->sched_runtime; + u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0; + int cpus, err = -1; - if (val == CPUFREQ_REMOVE_POLICY) { - update_min_max_capacity(); + if (new_bw == p->dl.dl_bw) return 0; - } - - for_each_cpu(i, policy->related_cpus) { - cpumask_copy(&cpu_rq(i)->freq_domain_cpumask, - policy->related_cpus); - orig_max_freq = cpu_rq(i)->max_freq; - cpu_rq(i)->min_freq = policy->min; - cpu_rq(i)->max_freq = policy->max; - cpu_rq(i)->max_possible_freq = policy->cpuinfo.max_freq; - } - - max_possible_freq = max(max_possible_freq, policy->cpuinfo.max_freq); - if (min_max_freq == 1) - min_max_freq = UINT_MAX; - min_max_freq = min(min_max_freq, policy->cpuinfo.max_freq); - BUG_ON(!min_max_freq); - BUG_ON(!policy->max); - - /* - * A changed min_max_freq or max_possible_freq (possible during bootup) - * needs to trigger re-computation of load_scale_factor and capacity for - * all possible cpus (even those offline). It also needs to trigger - * re-computation of nr_big/small_task count on all online cpus. - * - * A changed rq->max_freq otoh needs to trigger re-computation of - * load_scale_factor and capacity for just the cluster of cpus involved. - * Since small task definition depends on max_load_scale_factor, a - * changed load_scale_factor of one cluster could influence small_task - * classification of tasks in another cluster. Hence a changed - * rq->max_freq will need to trigger re-computation of nr_big/small_task - * count on all online cpus. - * - * While it should be sufficient for nr_big/small_tasks to be - * re-computed for only online cpus, we have inadequate context - * information here (in policy notifier) with regard to hotplug-safety - * context in which notification is issued. As a result, we can't use - * get_online_cpus() here, as it can lead to deadlock. Until cpufreq is - * fixed up to issue notification always in hotplug-safe context, - * re-compute nr_big/small_task for all possible cpus. - */ - - if (orig_min_max_freq != min_max_freq || - orig_max_possible_freq != max_possible_freq) { - cpus = cpu_possible_mask; - update_max = 1; - } /* - * Changed load_scale_factor can trigger reclassification of tasks as - * big or small. Make this change "atomic" so that tasks are accounted - * properly due to changed load_scale_factor + * Either if a task, enters, leave, or stays -deadline but changes + * its parameters, we may need to update accordingly the total + * allocated bandwidth of the container. */ - pre_big_small_task_count_change(cpu_possible_mask); - for_each_cpu(i, cpus) { - struct rq *rq = cpu_rq(i); - - rq->capacity = compute_capacity(i); - rq->load_scale_factor = compute_load_scale_factor(i); - - if (update_max) { - u64 mpc, mplsf; - - mpc = div_u64(((u64) rq->capacity) * - rq->max_possible_freq, rq->max_freq); - rq->max_possible_capacity = (int) mpc; - - mplsf = div_u64(((u64) rq->load_scale_factor) * - rq->max_possible_freq, rq->max_freq); - - if (mpc > highest_mpc) { - highest_mpc = mpc; - cpumask_clear(&mpc_mask); - cpumask_set_cpu(i, &mpc_mask); - } else if (mpc == highest_mpc) { - cpumask_set_cpu(i, &mpc_mask); - } - - if (mplsf > highest_mplsf) - highest_mplsf = mplsf; - } - } - - if (update_max) { - max_possible_capacity = highest_mpc; - max_load_scale_factor = highest_mplsf; - } - - __update_min_max_capacity(); - post_big_small_task_count_change(cpu_possible_mask); + raw_spin_lock(&dl_b->lock); + cpus = dl_bw_cpus(task_cpu(p)); + if (dl_policy(policy) && !task_has_dl_policy(p) && + !__dl_overflow(dl_b, cpus, 0, new_bw)) { + __dl_add(dl_b, new_bw); + err = 0; + } else if (dl_policy(policy) && task_has_dl_policy(p) && + !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) { + __dl_clear(dl_b, p->dl.dl_bw); + __dl_add(dl_b, new_bw); + err = 0; + } else if (!dl_policy(policy) && task_has_dl_policy(p)) { + __dl_clear(dl_b, p->dl.dl_bw); + err = 0; + } + raw_spin_unlock(&dl_b->lock); - return 0; + return err; } -static int cpufreq_notifier_trans(struct notifier_block *nb, - unsigned long val, void *data) +extern void init_dl_bw(struct dl_bw *dl_b); + +/* + * wake_up_new_task - wake up a newly created task for the first time. + * + * This function will do some initial scheduler statistics housekeeping + * that must be done for every newly created context, then puts the task + * on the runqueue and wakes it. + */ +void wake_up_new_task(struct task_struct *p) { - struct cpufreq_freqs *freq = (struct cpufreq_freqs *)data; - unsigned int cpu = freq->cpu, new_freq = freq->new; unsigned long flags; - int i; - - if (val != CPUFREQ_POSTCHANGE) - return 0; - - BUG_ON(!new_freq); - - if (cpu_rq(cpu)->cur_freq == new_freq) - return 0; + struct rq *rq; - for_each_cpu(i, &cpu_rq(cpu)->freq_domain_cpumask) { - struct rq *rq = cpu_rq(i); - raw_spin_lock_irqsave(&rq->lock, flags); - update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), 0); - rq->cur_freq = new_freq; - raw_spin_unlock_irqrestore(&rq->lock, flags); - } + raw_spin_lock_irqsave(&p->pi_lock, flags); + /* Initialize new task's runnable average */ + init_task_runnable_average(p); +#ifdef CONFIG_SMP + /* + * Fork balancing, do it here and not earlier because: + * - cpus_allowed can change in the fork path + * - any previously selected cpu might disappear through hotplug + */ + set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0)); +#endif - return 0; + rq = __task_rq_lock(p); + activate_task(rq, p, ENQUEUE_WAKEUP_NEW); + p->on_rq = TASK_ON_RQ_QUEUED; + trace_sched_wakeup_new(p, true); + check_preempt_curr(rq, p, WF_FORK); +#ifdef CONFIG_SMP + if (p->sched_class->task_woken) + p->sched_class->task_woken(rq, p); +#endif + task_rq_unlock(rq, p, &flags); } -static struct notifier_block notifier_policy_block = { - .notifier_call = cpufreq_notifier_policy -}; - -static struct notifier_block notifier_trans_block = { - .notifier_call = cpufreq_notifier_trans -}; +#ifdef CONFIG_PREEMPT_NOTIFIERS -static int register_sched_callback(void) +/** + * preempt_notifier_register - tell me when current is being preempted & rescheduled + * @notifier: notifier struct to register + */ +void preempt_notifier_register(struct preempt_notifier *notifier) { - int ret; - - if (!sched_enable_hmp) - return 0; - - ret = cpufreq_register_notifier(¬ifier_policy_block, - CPUFREQ_POLICY_NOTIFIER); - - if (!ret) - ret = cpufreq_register_notifier(¬ifier_trans_block, - CPUFREQ_TRANSITION_NOTIFIER); - - return 0; + hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); } +EXPORT_SYMBOL_GPL(preempt_notifier_register); -/* - * cpufreq callbacks can be registered at core_initcall or later time. - * Any registration done prior to that is "forgotten" by cpufreq. See - * initialization of variable init_cpufreq_transition_notifier_list_called - * for further information. +/** + * preempt_notifier_unregister - no longer interested in preemption notifications + * @notifier: notifier struct to unregister + * + * This is safe to call from within a preemption notifier. */ -core_initcall(register_sched_callback); - -static u64 orig_mark_start(struct task_struct *p) +void preempt_notifier_unregister(struct preempt_notifier *notifier) { - return p->ravg.mark_start; + hlist_del(¬ifier->link); } +EXPORT_SYMBOL_GPL(preempt_notifier_unregister); -static void restore_orig_mark_start(struct task_struct *p, u64 mark_start) +static void fire_sched_in_preempt_notifiers(struct task_struct *curr) { - p->ravg.mark_start = mark_start; -} + struct preempt_notifier *notifier; -/* - * Note down when task started running on a cpu. This information will be handy - * to avoid "too" frequent task migrations for a running task on account of - * power. - */ -static inline void note_run_start(struct task_struct *p, u64 wallclock) -{ - p->run_start = wallclock; + hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) + notifier->ops->sched_in(notifier, raw_smp_processor_id()); } -#else /* CONFIG_SCHED_HMP */ - -static inline void fixup_busy_time(struct task_struct *p, int new_cpu) { } - -static inline int -heavy_task_wakeup(struct task_struct *p, struct rq *rq, int event) -{ - return 0; -} - -static inline void -update_task_ravg(struct task_struct *p, struct rq *rq, - int event, u64 wallclock, u64 irqtime) -{ -} - -static inline void init_cpu_efficiency(void) {} - -static inline void mark_task_starting(struct task_struct *p) {} - -static inline void set_window_start(struct rq *rq) {} - -static inline void migrate_sync_cpu(int cpu) {} - -static inline u64 orig_mark_start(struct task_struct *p) { return 0; } - -static inline void -restore_orig_mark_start(struct task_struct *p, u64 mark_start) -{ -} - -static inline void note_run_start(struct task_struct *p, u64 wallclock) { } - -#endif /* CONFIG_SCHED_HMP */ - -#ifdef CONFIG_SMP -void set_task_cpu(struct task_struct *p, unsigned int new_cpu) -{ -#ifdef CONFIG_SCHED_DEBUG - /* - * We should never call set_task_cpu() on a blocked task, - * ttwu() will sort out the placement. - */ - WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING && - !(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE)); - -#ifdef CONFIG_LOCKDEP - /* - * The caller should hold either p->pi_lock or rq->lock, when changing - * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. - * - * sched_move_task() holds both and thus holding either pins the cgroup, - * see task_group(). - * - * Furthermore, all task_rq users should acquire both locks, see - * task_rq_lock(). - */ - WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || - lockdep_is_held(&task_rq(p)->lock))); -#endif -#endif - - trace_sched_migrate_task(p, new_cpu, pct_task_load(p)); - - note_run_start(p, sched_clock()); - - if (task_cpu(p) != new_cpu) { - struct task_migration_notifier tmn; - - if (p->sched_class->migrate_task_rq) - p->sched_class->migrate_task_rq(p, new_cpu); - p->se.nr_migrations++; - perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0); - - tmn.task = p; - tmn.from_cpu = task_cpu(p); - tmn.to_cpu = new_cpu; - - atomic_notifier_call_chain(&task_migration_notifier, 0, &tmn); - - fixup_busy_time(p, new_cpu); - } - - __set_task_cpu(p, new_cpu); -} - -struct migration_arg { - struct task_struct *task; - int dest_cpu; -}; - -static int migration_cpu_stop(void *data); - -/* - * wait_task_inactive - wait for a thread to unschedule. - * - * If @match_state is nonzero, it's the @p->state value just checked and - * not expected to change. If it changes, i.e. @p might have woken up, - * then return zero. When we succeed in waiting for @p to be off its CPU, - * we return a positive number (its total switch count). If a second call - * a short while later returns the same number, the caller can be sure that - * @p has remained unscheduled the whole time. - * - * The caller must ensure that the task *will* unschedule sometime soon, - * else this function might spin for a *long* time. This function can't - * be called with interrupts off, or it may introduce deadlock with - * smp_call_function() if an IPI is sent by the same process we are - * waiting to become inactive. - */ -unsigned long wait_task_inactive(struct task_struct *p, long match_state) -{ - unsigned long flags; - int running, on_rq; - unsigned long ncsw; - struct rq *rq; - - for (;;) { - /* - * We do the initial early heuristics without holding - * any task-queue locks at all. We'll only try to get - * the runqueue lock when things look like they will - * work out! - */ - rq = task_rq(p); - - /* - * If the task is actively running on another CPU - * still, just relax and busy-wait without holding - * any locks. - * - * NOTE! Since we don't hold any locks, it's not - * even sure that "rq" stays as the right runqueue! - * But we don't care, since "task_running()" will - * return false if the runqueue has changed and p - * is actually now running somewhere else! - */ - while (task_running(rq, p)) { - if (match_state && unlikely(p->state != match_state)) - return 0; - cpu_relax(); - } - - /* - * Ok, time to look more closely! We need the rq - * lock now, to be *sure*. If we're wrong, we'll - * just go back and repeat. - */ - rq = task_rq_lock(p, &flags); - trace_sched_wait_task(p); - running = task_running(rq, p); - on_rq = p->on_rq; - ncsw = 0; - if (!match_state || p->state == match_state) - ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ - task_rq_unlock(rq, p, &flags); - - /* - * If it changed from the expected state, bail out now. - */ - if (unlikely(!ncsw)) - break; - - /* - * Was it really running after all now that we - * checked with the proper locks actually held? - * - * Oops. Go back and try again.. - */ - if (unlikely(running)) { - cpu_relax(); - continue; - } - - /* - * It's not enough that it's not actively running, - * it must be off the runqueue _entirely_, and not - * preempted! - * - * So if it was still runnable (but just not actively - * running right now), it's preempted, and we should - * yield - it could be a while. - */ - if (unlikely(on_rq)) { - ktime_t to = ktime_set(0, NSEC_PER_MSEC); - - set_current_state(TASK_UNINTERRUPTIBLE); - schedule_hrtimeout(&to, HRTIMER_MODE_REL); - continue; - } - - /* - * Ahh, all good. It wasn't running, and it wasn't - * runnable, which means that it will never become - * running in the future either. We're all done! - */ - break; - } - - return ncsw; -} - -/*** - * kick_process - kick a running thread to enter/exit the kernel - * @p: the to-be-kicked thread - * - * Cause a process which is running on another CPU to enter - * kernel-mode, without any delay. (to get signals handled.) - * - * NOTE: this function doesn't have to take the runqueue lock, - * because all it wants to ensure is that the remote task enters - * the kernel. If the IPI races and the task has been migrated - * to another CPU then no harm is done and the purpose has been - * achieved as well. - */ -void kick_process(struct task_struct *p) -{ - int cpu; - - preempt_disable(); - cpu = task_cpu(p); - if ((cpu != smp_processor_id()) && task_curr(p)) - smp_send_reschedule(cpu); - preempt_enable(); -} -EXPORT_SYMBOL_GPL(kick_process); -#endif /* CONFIG_SMP */ - -#ifdef CONFIG_SMP -/* - * ->cpus_allowed is protected by both rq->lock and p->pi_lock - */ -static int select_fallback_rq(int cpu, struct task_struct *p) -{ - int nid = cpu_to_node(cpu); - const struct cpumask *nodemask = NULL; - enum { cpuset, possible, fail } state = cpuset; - int dest_cpu; - - /* - * If the node that the cpu is on has been offlined, cpu_to_node() - * will return -1. There is no cpu on the node, and we should - * select the cpu on the other node. - */ - if (nid != -1) { - nodemask = cpumask_of_node(nid); - - /* Look for allowed, online CPU in same node. */ - for_each_cpu(dest_cpu, nodemask) { - if (!cpu_online(dest_cpu)) - continue; - if (!cpu_active(dest_cpu)) - continue; - if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) - return dest_cpu; - } - } - - for (;;) { - /* Any allowed, online CPU? */ - for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) { - if (!cpu_online(dest_cpu)) - continue; - if (!cpu_active(dest_cpu)) - continue; - goto out; - } - - switch (state) { - case cpuset: - /* No more Mr. Nice Guy. */ - cpuset_cpus_allowed_fallback(p); - state = possible; - break; - - case possible: - do_set_cpus_allowed(p, cpu_possible_mask); - state = fail; - break; - - case fail: - BUG(); - break; - } - } - -out: - if (state != cpuset) { - /* - * Don't tell them about moving exiting tasks or - * kernel threads (both mm NULL), since they never - * leave kernel. - */ - if (p->mm && printk_ratelimit()) { - printk_deferred("process %d (%s) no longer affine to cpu%d\n", - task_pid_nr(p), p->comm, cpu); - } - } - - return dest_cpu; -} - -/* - * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable. - */ -static inline -int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags) -{ - int cpu = p->sched_class->select_task_rq(p, sd_flags, wake_flags); - - /* - * In order not to call set_task_cpu() on a blocking task we need - * to rely on ttwu() to place the task on a valid ->cpus_allowed - * cpu. - * - * Since this is common to all placement strategies, this lives here. - * - * [ this allows ->select_task() to simply return task_cpu(p) and - * not worry about this generic constraint ] - */ - if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) || - !cpu_online(cpu))) - cpu = select_fallback_rq(task_cpu(p), p); - - return cpu; -} - -static void update_avg(u64 *avg, u64 sample) -{ - s64 diff = sample - *avg; - *avg += diff >> 3; -} -#endif - -static void -ttwu_stat(struct task_struct *p, int cpu, int wake_flags) -{ -#ifdef CONFIG_SCHEDSTATS - struct rq *rq = this_rq(); - -#ifdef CONFIG_SMP - int this_cpu = smp_processor_id(); - - if (cpu == this_cpu) { - schedstat_inc(rq, ttwu_local); - schedstat_inc(p, se.statistics.nr_wakeups_local); - } else { - struct sched_domain *sd; - - schedstat_inc(p, se.statistics.nr_wakeups_remote); - rcu_read_lock(); - for_each_domain(this_cpu, sd) { - if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { - schedstat_inc(sd, ttwu_wake_remote); - break; - } - } - rcu_read_unlock(); - } - - if (wake_flags & WF_MIGRATED) - schedstat_inc(p, se.statistics.nr_wakeups_migrate); - -#endif /* CONFIG_SMP */ - - schedstat_inc(rq, ttwu_count); - schedstat_inc(p, se.statistics.nr_wakeups); - - if (wake_flags & WF_SYNC) - schedstat_inc(p, se.statistics.nr_wakeups_sync); - -#endif /* CONFIG_SCHEDSTATS */ -} - -static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags) -{ - activate_task(rq, p, en_flags); - p->on_rq = 1; - - /* if a worker is waking up, notify workqueue */ - if (p->flags & PF_WQ_WORKER) - wq_worker_waking_up(p, cpu_of(rq)); -} - -/* - * Mark the task runnable and perform wakeup-preemption. - */ -static void -ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) -{ - check_preempt_curr(rq, p, wake_flags); - trace_sched_wakeup(p, true); - - p->state = TASK_RUNNING; -#ifdef CONFIG_SMP - if (p->sched_class->task_woken) - p->sched_class->task_woken(rq, p); - - if (rq->idle_stamp) { - u64 delta = rq->clock - rq->idle_stamp; - u64 max = 2*sysctl_sched_migration_cost; - - if (delta > max) - rq->avg_idle = max; - else - update_avg(&rq->avg_idle, delta); - rq->idle_stamp = 0; - } -#endif -} - -static void -ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags) -{ -#ifdef CONFIG_SMP - if (p->sched_contributes_to_load) - rq->nr_uninterruptible--; -#endif - - ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING); - ttwu_do_wakeup(rq, p, wake_flags); -} - -/* - * Called in case the task @p isn't fully descheduled from its runqueue, - * in this case we must do a remote wakeup. Its a 'light' wakeup though, - * since all we need to do is flip p->state to TASK_RUNNING, since - * the task is still ->on_rq. - */ -static int ttwu_remote(struct task_struct *p, int wake_flags) -{ - struct rq *rq; - int ret = 0; - - rq = __task_rq_lock(p); - if (p->on_rq) { - ttwu_do_wakeup(rq, p, wake_flags); - ret = 1; - } - __task_rq_unlock(rq); - - return ret; -} - -#ifdef CONFIG_SMP -static void sched_ttwu_pending(void) -{ - struct rq *rq = this_rq(); - struct llist_node *llist = llist_del_all(&rq->wake_list); - struct task_struct *p; - - raw_spin_lock(&rq->lock); - - while (llist) { - p = llist_entry(llist, struct task_struct, wake_entry); - llist = llist_next(llist); - ttwu_do_activate(rq, p, 0); - } - - raw_spin_unlock(&rq->lock); -} - -void scheduler_ipi(void) -{ - int cpu = smp_processor_id(); - - if (llist_empty(&this_rq()->wake_list) - && !tick_nohz_full_cpu(cpu) - && !got_nohz_idle_kick() - && !got_boost_kick()) - return; - - if (got_boost_kick()) { - struct rq *rq = cpu_rq(cpu); - - if (rq->curr->sched_class == &fair_sched_class) - check_for_migration(rq, rq->curr); - clear_boost_kick(cpu); - } - - - /* - * Not all reschedule IPI handlers call irq_enter/irq_exit, since - * traditionally all their work was done from the interrupt return - * path. Now that we actually do some work, we need to make sure - * we do call them. - * - * Some archs already do call them, luckily irq_enter/exit nest - * properly. - * - * Arguably we should visit all archs and update all handlers, - * however a fair share of IPIs are still resched only so this would - * somewhat pessimize the simple resched case. - */ - irq_enter(); - tick_nohz_full_check(); - sched_ttwu_pending(); - - /* - * Check if someone kicked us for doing the nohz idle load balance. - */ - if (unlikely(got_nohz_idle_kick())) { - this_rq()->idle_balance = 1; - raise_softirq_irqoff(SCHED_SOFTIRQ); - } - irq_exit(); -} - -static void ttwu_queue_remote(struct task_struct *p, int cpu) -{ - if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) - smp_send_reschedule(cpu); -} - -bool cpus_share_cache(int this_cpu, int that_cpu) -{ - return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); -} -#endif /* CONFIG_SMP */ - -static void ttwu_queue(struct task_struct *p, int cpu) -{ - struct rq *rq = cpu_rq(cpu); - -#if defined(CONFIG_SMP) - if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { - sched_clock_cpu(cpu); /* sync clocks x-cpu */ - ttwu_queue_remote(p, cpu); - return; - } -#endif - - raw_spin_lock(&rq->lock); - ttwu_do_activate(rq, p, 0); - raw_spin_unlock(&rq->lock); -} - -__read_mostly unsigned int sysctl_sched_wakeup_load_threshold = 110; - -/** - * try_to_wake_up - wake up a thread - * @p: the thread to be awakened - * @state: the mask of task states that can be woken - * @wake_flags: wake modifier flags (WF_*) - * - * Put it on the run-queue if it's not already there. The "current" - * thread is always on the run-queue (except when the actual - * re-schedule is in progress), and as such you're allowed to do - * the simpler "current->state = TASK_RUNNING" to mark yourself - * runnable without the overhead of this. - * - * Returns %true if @p was woken up, %false if it was already running - * or @state didn't match @p's state. - */ -static int -try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) -{ - unsigned long flags; - int cpu, success = 0; - unsigned long src_cpu; - int notify = 0; - struct migration_notify_data mnd; - int heavy_task = 0; -#ifdef CONFIG_SMP - struct rq *rq; - u64 wallclock; -#endif - - /* - * If we are going to wake up a thread waiting for CONDITION we - * need to ensure that CONDITION=1 done by the caller can not be - * reordered with p->state check below. This pairs with mb() in - * set_current_state() the waiting thread does. - */ - smp_mb__before_spinlock(); - raw_spin_lock_irqsave(&p->pi_lock, flags); - src_cpu = cpu = task_cpu(p); - - if (!(p->state & state)) - goto out; - - success = 1; /* we're going to change ->state */ - - /* - * Ensure we load p->on_rq _after_ p->state, otherwise it would - * be possible to, falsely, observe p->on_rq == 0 and get stuck - * in smp_cond_load_acquire() below. - * - * sched_ttwu_pending() try_to_wake_up() - * [S] p->on_rq = 1; [L] P->state - * UNLOCK rq->lock -----. - * \ - * +--- RMB - * schedule() / - * LOCK rq->lock -----' - * UNLOCK rq->lock - * - * [task p] - * [S] p->state = UNINTERRUPTIBLE [L] p->on_rq - * - * Pairs with the UNLOCK+LOCK on rq->lock from the - * last wakeup of our task and the schedule that got our task - * current. - */ - smp_rmb(); - if (p->on_rq && ttwu_remote(p, wake_flags)) - goto stat; - -#ifdef CONFIG_SMP - /* - * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be - * possible to, falsely, observe p->on_cpu == 0. - * - * One must be running (->on_cpu == 1) in order to remove oneself - * from the runqueue. - * - * [S] ->on_cpu = 1; [L] ->on_rq - * UNLOCK rq->lock - * RMB - * LOCK rq->lock - * [S] ->on_rq = 0; [L] ->on_cpu - * - * Pairs with the full barrier implied in the UNLOCK+LOCK on rq->lock - * from the consecutive calls to schedule(); the first switching to our - * task, the second putting it to sleep. - */ - smp_rmb(); - - /* - * If the owning (remote) cpu is still in the middle of schedule() with - * this task as prev, wait until its done referencing the task. - */ - while (p->on_cpu) - cpu_relax(); - /* - * Pairs with the smp_wmb() in finish_lock_switch(). - */ - smp_rmb(); - - rq = cpu_rq(task_cpu(p)); - - raw_spin_lock(&rq->lock); - wallclock = sched_clock(); - update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); - heavy_task = heavy_task_wakeup(p, rq, TASK_WAKE); - update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); - raw_spin_unlock(&rq->lock); - - p->sched_contributes_to_load = !!task_contributes_to_load(p); - p->state = TASK_WAKING; - - if (p->sched_class->task_waking) - p->sched_class->task_waking(p); - - cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags); - - /* Refresh src_cpu as it could have changed since we last read it */ - src_cpu = task_cpu(p); - if (src_cpu != cpu) { - wake_flags |= WF_MIGRATED; - set_task_cpu(p, cpu); - } else { - note_run_start(p, wallclock); - } -#endif /* CONFIG_SMP */ - - ttwu_queue(p, cpu); -stat: - ttwu_stat(p, cpu, wake_flags); - - if (task_notify_on_migrate(p)) { - mnd.src_cpu = src_cpu; - mnd.dest_cpu = cpu; - mnd.load = pct_task_load(p); - - /* - * Call the migration notifier with mnd for foreground task - * migrations as well as for wakeups if their load is above - * sysctl_sched_wakeup_load_threshold. This would prompt the - * cpu-boost to boost the CPU frequency on wake up of a heavy - * weight foreground task - */ - if ((src_cpu != cpu) || (mnd.load > - sysctl_sched_wakeup_load_threshold)) - notify = 1; - } - -out: - raw_spin_unlock_irqrestore(&p->pi_lock, flags); - - if (notify) - atomic_notifier_call_chain(&migration_notifier_head, - 0, (void *)&mnd); - - if (!same_freq_domain(src_cpu, cpu)) { - check_for_freq_change(cpu_rq(cpu)); - check_for_freq_change(cpu_rq(src_cpu)); - } else if (heavy_task) - check_for_freq_change(cpu_rq(cpu)); - - return success; -} - -/** - * try_to_wake_up_local - try to wake up a local task with rq lock held - * @p: the thread to be awakened - * - * Put @p on the run-queue if it's not already there. The caller must - * ensure that this_rq() is locked, @p is bound to this_rq() and not - * the current task. - */ -static void try_to_wake_up_local(struct task_struct *p) -{ - struct rq *rq = task_rq(p); - - if (rq != this_rq() || p == current) { - printk_deferred("%s: Failed to wakeup task %d (%s), rq = %p, this_rq = %p, p = %p, current = %p\n", - __func__, task_pid_nr(p), p->comm, rq, - this_rq(), p, current); - return; - } - - lockdep_assert_held(&rq->lock); - - if (!raw_spin_trylock(&p->pi_lock)) { - raw_spin_unlock(&rq->lock); - raw_spin_lock(&p->pi_lock); - raw_spin_lock(&rq->lock); - } - - if (!(p->state & TASK_NORMAL)) - goto out; - - if (!p->on_rq) { - u64 wallclock = sched_clock(); - - update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); - update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); - ttwu_activate(rq, p, ENQUEUE_WAKEUP); - } - - ttwu_do_wakeup(rq, p, 0); - ttwu_stat(p, smp_processor_id(), 0); -out: - raw_spin_unlock(&p->pi_lock); - /* Todo : Send cpufreq notifier */ -} - -/** - * wake_up_process - Wake up a specific process - * @p: The process to be woken up. - * - * Attempt to wake up the nominated process and move it to the set of runnable - * processes. Returns 1 if the process was woken up, 0 if it was already - * running. - * - * It may be assumed that this function implies a write memory barrier before - * changing the task state if and only if any tasks are woken up. - */ -int wake_up_process(struct task_struct *p) -{ - return try_to_wake_up(p, TASK_NORMAL, 0); -} -EXPORT_SYMBOL(wake_up_process); - -int wake_up_state(struct task_struct *p, unsigned int state) -{ - return try_to_wake_up(p, state, 0); -} - -/* - * Perform scheduler related setup for a newly forked process p. - * p is forked by current. - * - * __sched_fork() is basic setup used by init_idle() too: - */ -static void __sched_fork(struct task_struct *p) -{ - p->on_rq = 0; - - p->se.on_rq = 0; - p->se.exec_start = 0; - p->se.sum_exec_runtime = 0; - p->se.prev_sum_exec_runtime = 0; - p->se.nr_migrations = 0; - p->se.vruntime = 0; - - INIT_LIST_HEAD(&p->se.group_node); - -/* - * Load-tracking only depends on SMP, FAIR_GROUP_SCHED dependency below may be - * removed when useful for applications beyond shares distribution (e.g. - * load-balance). - */ -#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED) - init_new_task_load(p); -#endif -#ifdef CONFIG_SCHEDSTATS - memset(&p->se.statistics, 0, sizeof(p->se.statistics)); -#endif - - INIT_LIST_HEAD(&p->rt.run_list); - -#ifdef CONFIG_PREEMPT_NOTIFIERS - INIT_HLIST_HEAD(&p->preempt_notifiers); -#endif - -#ifdef CONFIG_NUMA_BALANCING - if (p->mm && atomic_read(&p->mm->mm_users) == 1) { - p->mm->numa_next_scan = jiffies; - p->mm->numa_next_reset = jiffies; - p->mm->numa_scan_seq = 0; - } - - p->node_stamp = 0ULL; - p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0; - p->numa_migrate_seq = p->mm ? p->mm->numa_scan_seq - 1 : 0; - p->numa_scan_period = sysctl_numa_balancing_scan_delay; - p->numa_work.next = &p->numa_work; -#endif /* CONFIG_NUMA_BALANCING */ -} - -#ifdef CONFIG_NUMA_BALANCING -#ifdef CONFIG_SCHED_DEBUG -void set_numabalancing_state(bool enabled) -{ - if (enabled) - sched_feat_set("NUMA"); - else - sched_feat_set("NO_NUMA"); -} -#else -__read_mostly bool numabalancing_enabled; - -void set_numabalancing_state(bool enabled) -{ - numabalancing_enabled = enabled; -} -#endif /* CONFIG_SCHED_DEBUG */ -#endif /* CONFIG_NUMA_BALANCING */ - -/* - * fork()/clone()-time setup: - */ -void sched_fork(struct task_struct *p) -{ - unsigned long flags; - int cpu = get_cpu(); - - __sched_fork(p); - /* - * We mark the process as running here. This guarantees that - * nobody will actually run it, and a signal or other external - * event cannot wake it up and insert it on the runqueue either. - */ - p->state = TASK_RUNNING; - - /* - * Make sure we do not leak PI boosting priority to the child. - */ - p->prio = current->normal_prio; - - /* - * Revert to default priority/policy on fork if requested. - */ - if (unlikely(p->sched_reset_on_fork)) { - if (task_has_rt_policy(p)) { - p->policy = SCHED_NORMAL; - p->static_prio = NICE_TO_PRIO(0); - p->rt_priority = 0; - } else if (PRIO_TO_NICE(p->static_prio) < 0) - p->static_prio = NICE_TO_PRIO(0); - - p->prio = p->normal_prio = __normal_prio(p); - set_load_weight(p); - - /* - * We don't need the reset flag anymore after the fork. It has - * fulfilled its duty: - */ - p->sched_reset_on_fork = 0; - } - - if (!rt_prio(p->prio)) - p->sched_class = &fair_sched_class; - - if (p->sched_class->task_fork) - p->sched_class->task_fork(p); - - /* - * The child is not yet in the pid-hash so no cgroup attach races, - * and the cgroup is pinned to this child due to cgroup_fork() - * is ran before sched_fork(). - * - * Silence PROVE_RCU. - */ - raw_spin_lock_irqsave(&p->pi_lock, flags); - set_task_cpu(p, cpu); - raw_spin_unlock_irqrestore(&p->pi_lock, flags); - -#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) - if (likely(sched_info_on())) - memset(&p->sched_info, 0, sizeof(p->sched_info)); -#endif -#if defined(CONFIG_SMP) - p->on_cpu = 0; -#endif -#ifdef CONFIG_PREEMPT_COUNT - /* Want to start with kernel preemption disabled. */ - task_thread_info(p)->preempt_count = 1; -#endif -#ifdef CONFIG_SMP - plist_node_init(&p->pushable_tasks, MAX_PRIO); -#endif - - put_cpu(); -} - -/* - * wake_up_new_task - wake up a newly created task for the first time. - * - * This function will do some initial scheduler statistics housekeeping - * that must be done for every newly created context, then puts the task - * on the runqueue and wakes it. - */ -void wake_up_new_task(struct task_struct *p) -{ - unsigned long flags; - struct rq *rq; - - raw_spin_lock_irqsave(&p->pi_lock, flags); -#ifdef CONFIG_SMP - /* - * Fork balancing, do it here and not earlier because: - * - cpus_allowed can change in the fork path - * - any previously selected cpu might disappear through hotplug - */ - set_task_cpu(p, select_task_rq(p, SD_BALANCE_FORK, 0)); -#endif - - rq = __task_rq_lock(p); - mark_task_starting(p); - activate_task(rq, p, 0); - p->on_rq = 1; - trace_sched_wakeup_new(p, true); - check_preempt_curr(rq, p, WF_FORK); -#ifdef CONFIG_SMP - if (p->sched_class->task_woken) - p->sched_class->task_woken(rq, p); -#endif - task_rq_unlock(rq, p, &flags); -} - -#ifdef CONFIG_PREEMPT_NOTIFIERS - -/** - * preempt_notifier_register - tell me when current is being preempted & rescheduled - * @notifier: notifier struct to register - */ -void preempt_notifier_register(struct preempt_notifier *notifier) -{ - hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); -} -EXPORT_SYMBOL_GPL(preempt_notifier_register); - -/** - * preempt_notifier_unregister - no longer interested in preemption notifications - * @notifier: notifier struct to unregister - * - * This is safe to call from within a preemption notifier. - */ -void preempt_notifier_unregister(struct preempt_notifier *notifier) -{ - hlist_del(¬ifier->link); -} -EXPORT_SYMBOL_GPL(preempt_notifier_unregister); - -static void fire_sched_in_preempt_notifiers(struct task_struct *curr) -{ - struct preempt_notifier *notifier; - - hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) - notifier->ops->sched_in(notifier, raw_smp_processor_id()); -} - -static void -fire_sched_out_preempt_notifiers(struct task_struct *curr, - struct task_struct *next) -{ - struct preempt_notifier *notifier; - - hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) - notifier->ops->sched_out(notifier, next); -} - -#else /* !CONFIG_PREEMPT_NOTIFIERS */ - -static void fire_sched_in_preempt_notifiers(struct task_struct *curr) -{ -} - -static void -fire_sched_out_preempt_notifiers(struct task_struct *curr, - struct task_struct *next) -{ -} - -#endif /* CONFIG_PREEMPT_NOTIFIERS */ - -/** - * prepare_task_switch - prepare to switch tasks - * @rq: the runqueue preparing to switch - * @prev: the current task that is being switched out - * @next: the task we are going to switch to. - * - * This is called with the rq lock held and interrupts off. It must - * be paired with a subsequent finish_task_switch after the context - * switch. - * - * prepare_task_switch sets up locking and calls architecture specific - * hooks. - */ -static inline void -prepare_task_switch(struct rq *rq, struct task_struct *prev, - struct task_struct *next) -{ - trace_sched_switch(prev, next); -#ifdef CONFIG_ARCH_WANTS_CTXSW_LOGGING - dlog("%s: end trace at %llu\n", __func__, sched_clock()); -#endif - sched_info_switch(prev, next); - perf_event_task_sched_out(prev, next); - fire_sched_out_preempt_notifiers(prev, next); - prepare_lock_switch(rq, next); - prepare_arch_switch(next); -} - -/** - * finish_task_switch - clean up after a task-switch - * @rq: runqueue associated with task-switch - * @prev: the thread we just switched away from. - * - * finish_task_switch must be called after the context switch, paired - * with a prepare_task_switch call before the context switch. - * finish_task_switch will reconcile locking set up by prepare_task_switch, - * and do any other architecture-specific cleanup actions. - * - * Note that we may have delayed dropping an mm in context_switch(). If - * so, we finish that here outside of the runqueue lock. (Doing it - * with the lock held can cause deadlocks; see schedule() for - * details.) - */ -static void finish_task_switch(struct rq *rq, struct task_struct *prev) - __releases(rq->lock) -{ - struct mm_struct *mm = rq->prev_mm; - long prev_state; - - rq->prev_mm = NULL; - - /* - * A task struct has one reference for the use as "current". - * If a task dies, then it sets TASK_DEAD in tsk->state and calls - * schedule one last time. The schedule call will never return, and - * the scheduled task must drop that reference. - * The test for TASK_DEAD must occur while the runqueue locks are - * still held, otherwise prev could be scheduled on another cpu, die - * there before we look at prev->state, and then the reference would - * be dropped twice. - * Manfred Spraul - */ - prev_state = prev->state; - vtime_task_switch(prev); - finish_arch_switch(prev); - perf_event_task_sched_in(prev, current); - finish_lock_switch(rq, prev); - finish_arch_post_lock_switch(); - - fire_sched_in_preempt_notifiers(current); - if (mm) - mmdrop(mm); - if (unlikely(prev_state == TASK_DEAD)) { - /* - * Remove function-return probe instances associated with this - * task and put them back on the free list. - */ - kprobe_flush_task(prev); - put_task_struct(prev); - } - - tick_nohz_task_switch(current); -} - -#ifdef CONFIG_SMP - -/* assumes rq->lock is held */ -static inline void pre_schedule(struct rq *rq, struct task_struct *prev) -{ - if (prev->sched_class->pre_schedule) - prev->sched_class->pre_schedule(rq, prev); -} - -/* rq->lock is NOT held, but preemption is disabled */ -static inline void post_schedule(struct rq *rq) -{ - if (rq->post_schedule) { - unsigned long flags; - - raw_spin_lock_irqsave(&rq->lock, flags); - if (rq->curr->sched_class->post_schedule) - rq->curr->sched_class->post_schedule(rq); - raw_spin_unlock_irqrestore(&rq->lock, flags); - - rq->post_schedule = 0; - } -} - -#else - -static inline void pre_schedule(struct rq *rq, struct task_struct *p) -{ -} - -static inline void post_schedule(struct rq *rq) -{ -} - -#endif - -/** - * schedule_tail - first thing a freshly forked thread must call. - * @prev: the thread we just switched away from. - */ -asmlinkage void schedule_tail(struct task_struct *prev) - __releases(rq->lock) -{ - struct rq *rq = this_rq(); - - finish_task_switch(rq, prev); - - /* - * FIXME: do we need to worry about rq being invalidated by the - * task_switch? - */ - post_schedule(rq); - -#ifdef __ARCH_WANT_UNLOCKED_CTXSW - /* In this case, finish_task_switch does not reenable preemption */ - preempt_enable(); -#endif - if (current->set_child_tid) - put_user(task_pid_vnr(current), current->set_child_tid); -} - -/* - * context_switch - switch to the new MM and the new - * thread's register state. - */ -static inline void -context_switch(struct rq *rq, struct task_struct *prev, - struct task_struct *next) -{ - struct mm_struct *mm, *oldmm; -#ifdef CONFIG_ARCH_WANTS_CTXSW_LOGGING - u64 start, end; -#endif - - prepare_task_switch(rq, prev, next); - - mm = next->mm; - oldmm = prev->active_mm; - -#ifdef CONFIG_ARCH_WANTS_CTXSW_LOGGING - start = sched_clock(); - dlog("%s: n_mm: %p, finish pts at %llu\n", __func__, mm, start); -#endif - /* - * For paravirt, this is coupled with an exit in switch_to to - * combine the page table reload and the switch backend into - * one hypercall. - */ - arch_start_context_switch(prev); - - if (!mm) { - next->active_mm = oldmm; - atomic_inc(&oldmm->mm_count); - enter_lazy_tlb(oldmm, next); - } else - switch_mm(oldmm, mm, next); - - if (!prev->mm) { - prev->active_mm = NULL; - rq->prev_mm = oldmm; - } - -#ifdef CONFIG_ARCH_WANTS_CTXSW_LOGGING - end = sched_clock(); - dlog("%s: start task switch at %llu\n", __func__, end); - - if (end - start > MAX_CTXSW_LATENCY) - msm_trigger_wdog_bite(); -#endif - - /* - * Since the runqueue lock will be released by the next - * task (which is an invalid locking op but in the case - * of the scheduler it's an obvious special-case), so we - * do an early lockdep release here: - */ -#ifndef __ARCH_WANT_UNLOCKED_CTXSW - spin_release(&rq->lock.dep_map, 1, _THIS_IP_); -#endif - - context_tracking_task_switch(prev, next); - /* Here we just switch the register state and the stack. */ - switch_to(prev, next, prev); - - barrier(); - /* - * this_rq must be evaluated again because prev may have moved - * CPUs since it called schedule(), thus the 'rq' on its stack - * frame will be invalid. - */ - finish_task_switch(this_rq(), prev); -} - -/* - * nr_running and nr_context_switches: - * - * externally visible scheduler statistics: current number of runnable - * threads, total number of context switches performed since bootup. - */ -unsigned long nr_running(void) -{ - unsigned long i, sum = 0; - - for_each_online_cpu(i) - sum += cpu_rq(i)->nr_running; - - return sum; -} - -unsigned long long nr_context_switches(void) -{ - int i; - unsigned long long sum = 0; - - for_each_possible_cpu(i) - sum += cpu_rq(i)->nr_switches; - - return sum; -} - -unsigned long nr_iowait(void) -{ - unsigned long i, sum = 0; - - for_each_possible_cpu(i) - sum += atomic_read(&cpu_rq(i)->nr_iowait); - - return sum; -} - -unsigned long nr_iowait_cpu(int cpu) -{ - struct rq *this = cpu_rq(cpu); - return atomic_read(&this->nr_iowait); -} - -unsigned long this_cpu_load(void) -{ - struct rq *this = this_rq(); - return this->cpu_load[0]; -} - - -/* - * Global load-average calculations - * - * We take a distributed and async approach to calculating the global load-avg - * in order to minimize overhead. - * - * The global load average is an exponentially decaying average of nr_running + - * nr_uninterruptible. - * - * Once every LOAD_FREQ: - * - * nr_active = 0; - * for_each_possible_cpu(cpu) - * nr_active += cpu_of(cpu)->nr_running + cpu_of(cpu)->nr_uninterruptible; - * - * avenrun[n] = avenrun[0] * exp_n + nr_active * (1 - exp_n) - * - * Due to a number of reasons the above turns in the mess below: - * - * - for_each_possible_cpu() is prohibitively expensive on machines with - * serious number of cpus, therefore we need to take a distributed approach - * to calculating nr_active. - * - * \Sum_i x_i(t) = \Sum_i x_i(t) - x_i(t_0) | x_i(t_0) := 0 - * = \Sum_i { \Sum_j=1 x_i(t_j) - x_i(t_j-1) } - * - * So assuming nr_active := 0 when we start out -- true per definition, we - * can simply take per-cpu deltas and fold those into a global accumulate - * to obtain the same result. See calc_load_fold_active(). - * - * Furthermore, in order to avoid synchronizing all per-cpu delta folding - * across the machine, we assume 10 ticks is sufficient time for every - * cpu to have completed this task. - * - * This places an upper-bound on the IRQ-off latency of the machine. Then - * again, being late doesn't loose the delta, just wrecks the sample. - * - * - cpu_rq()->nr_uninterruptible isn't accurately tracked per-cpu because - * this would add another cross-cpu cacheline miss and atomic operation - * to the wakeup path. Instead we increment on whatever cpu the task ran - * when it went into uninterruptible state and decrement on whatever cpu - * did the wakeup. This means that only the sum of nr_uninterruptible over - * all cpus yields the correct result. - * - * This covers the NO_HZ=n code, for extra head-aches, see the comment below. - */ - -/* Variables and functions for calc_load */ -static atomic_long_t calc_load_tasks; -static unsigned long calc_load_update; -unsigned long avenrun[3]; -EXPORT_SYMBOL(avenrun); /* should be removed */ - -/** - * get_avenrun - get the load average array - * @loads: pointer to dest load array - * @offset: offset to add - * @shift: shift count to shift the result left - * - * These values are estimates at best, so no need for locking. - */ -void get_avenrun(unsigned long *loads, unsigned long offset, int shift) -{ - loads[0] = (avenrun[0] + offset) << shift; - loads[1] = (avenrun[1] + offset) << shift; - loads[2] = (avenrun[2] + offset) << shift; -} - -static long calc_load_fold_active(struct rq *this_rq) -{ - long nr_active, delta = 0; - - nr_active = this_rq->nr_running; - nr_active += (long) this_rq->nr_uninterruptible; - - if (nr_active != this_rq->calc_load_active) { - delta = nr_active - this_rq->calc_load_active; - this_rq->calc_load_active = nr_active; - } - - return delta; -} - -/* - * a1 = a0 * e + a * (1 - e) - */ -static unsigned long -calc_load(unsigned long load, unsigned long exp, unsigned long active) -{ - load *= exp; - load += active * (FIXED_1 - exp); - load += 1UL << (FSHIFT - 1); - return load >> FSHIFT; -} - -#ifdef CONFIG_NO_HZ_COMMON -/* - * Handle NO_HZ for the global load-average. - * - * Since the above described distributed algorithm to compute the global - * load-average relies on per-cpu sampling from the tick, it is affected by - * NO_HZ. - * - * The basic idea is to fold the nr_active delta into a global idle-delta upon - * entering NO_HZ state such that we can include this as an 'extra' cpu delta - * when we read the global state. - * - * Obviously reality has to ruin such a delightfully simple scheme: - * - * - When we go NO_HZ idle during the window, we can negate our sample - * contribution, causing under-accounting. - * - * We avoid this by keeping two idle-delta counters and flipping them - * when the window starts, thus separating old and new NO_HZ load. - * - * The only trick is the slight shift in index flip for read vs write. - * - * 0s 5s 10s 15s - * +10 +10 +10 +10 - * |-|-----------|-|-----------|-|-----------|-| - * r:0 0 1 1 0 0 1 1 0 - * w:0 1 1 0 0 1 1 0 0 - * - * This ensures we'll fold the old idle contribution in this window while - * accumlating the new one. - * - * - When we wake up from NO_HZ idle during the window, we push up our - * contribution, since we effectively move our sample point to a known - * busy state. - * - * This is solved by pushing the window forward, and thus skipping the - * sample, for this cpu (effectively using the idle-delta for this cpu which - * was in effect at the time the window opened). This also solves the issue - * of having to deal with a cpu having been in NOHZ idle for multiple - * LOAD_FREQ intervals. - * - * When making the ILB scale, we should try to pull this in as well. - */ -static atomic_long_t calc_load_idle[2]; -static int calc_load_idx; - -static inline int calc_load_write_idx(void) -{ - int idx = calc_load_idx; - - /* - * See calc_global_nohz(), if we observe the new index, we also - * need to observe the new update time. - */ - smp_rmb(); - - /* - * If the folding window started, make sure we start writing in the - * next idle-delta. - */ - if (!time_before(jiffies, calc_load_update)) - idx++; - - return idx & 1; -} - -static inline int calc_load_read_idx(void) -{ - return calc_load_idx & 1; -} - -void calc_load_enter_idle(void) -{ - struct rq *this_rq = this_rq(); - long delta; - - /* - * We're going into NOHZ mode, if there's any pending delta, fold it - * into the pending idle delta. - */ - delta = calc_load_fold_active(this_rq); - if (delta) { - int idx = calc_load_write_idx(); - atomic_long_add(delta, &calc_load_idle[idx]); - } -} - -void calc_load_exit_idle(void) -{ - struct rq *this_rq = this_rq(); - - /* - * If we're still before the sample window, we're done. - */ - if (time_before(jiffies, this_rq->calc_load_update)) - return; - - /* - * We woke inside or after the sample window, this means we're already - * accounted through the nohz accounting, so skip the entire deal and - * sync up for the next window. - */ - this_rq->calc_load_update = calc_load_update; - if (time_before(jiffies, this_rq->calc_load_update + 10)) - this_rq->calc_load_update += LOAD_FREQ; -} - -static long calc_load_fold_idle(void) -{ - int idx = calc_load_read_idx(); - long delta = 0; - - if (atomic_long_read(&calc_load_idle[idx])) - delta = atomic_long_xchg(&calc_load_idle[idx], 0); - - return delta; -} - -/** - * fixed_power_int - compute: x^n, in O(log n) time - * - * @x: base of the power - * @frac_bits: fractional bits of @x - * @n: power to raise @x to. - * - * By exploiting the relation between the definition of the natural power - * function: x^n := x*x*...*x (x multiplied by itself for n times), and - * the binary encoding of numbers used by computers: n := \Sum n_i * 2^i, - * (where: n_i \elem {0, 1}, the binary vector representing n), - * we find: x^n := x^(\Sum n_i * 2^i) := \Prod x^(n_i * 2^i), which is - * of course trivially computable in O(log_2 n), the length of our binary - * vector. - */ -static unsigned long -fixed_power_int(unsigned long x, unsigned int frac_bits, unsigned int n) -{ - unsigned long result = 1UL << frac_bits; - - if (n) for (;;) { - if (n & 1) { - result *= x; - result += 1UL << (frac_bits - 1); - result >>= frac_bits; - } - n >>= 1; - if (!n) - break; - x *= x; - x += 1UL << (frac_bits - 1); - x >>= frac_bits; - } - - return result; -} - -/* - * a1 = a0 * e + a * (1 - e) - * - * a2 = a1 * e + a * (1 - e) - * = (a0 * e + a * (1 - e)) * e + a * (1 - e) - * = a0 * e^2 + a * (1 - e) * (1 + e) - * - * a3 = a2 * e + a * (1 - e) - * = (a0 * e^2 + a * (1 - e) * (1 + e)) * e + a * (1 - e) - * = a0 * e^3 + a * (1 - e) * (1 + e + e^2) - * - * ... - * - * an = a0 * e^n + a * (1 - e) * (1 + e + ... + e^n-1) [1] - * = a0 * e^n + a * (1 - e) * (1 - e^n)/(1 - e) - * = a0 * e^n + a * (1 - e^n) - * - * [1] application of the geometric series: - * - * n 1 - x^(n+1) - * S_n := \Sum x^i = ------------- - * i=0 1 - x - */ -static unsigned long -calc_load_n(unsigned long load, unsigned long exp, - unsigned long active, unsigned int n) -{ - - return calc_load(load, fixed_power_int(exp, FSHIFT, n), active); -} - -/* - * NO_HZ can leave us missing all per-cpu ticks calling - * calc_load_account_active(), but since an idle CPU folds its delta into - * calc_load_tasks_idle per calc_load_account_idle(), all we need to do is fold - * in the pending idle delta if our idle period crossed a load cycle boundary. - * - * Once we've updated the global active value, we need to apply the exponential - * weights adjusted to the number of cycles missed. - */ -static void calc_global_nohz(void) -{ - long delta, active, n; - - if (!time_before(jiffies, calc_load_update + 10)) { - /* - * Catch-up, fold however many we are behind still - */ - delta = jiffies - calc_load_update - 10; - n = 1 + (delta / LOAD_FREQ); - - active = atomic_long_read(&calc_load_tasks); - active = active > 0 ? active * FIXED_1 : 0; - - avenrun[0] = calc_load_n(avenrun[0], EXP_1, active, n); - avenrun[1] = calc_load_n(avenrun[1], EXP_5, active, n); - avenrun[2] = calc_load_n(avenrun[2], EXP_15, active, n); - - calc_load_update += n * LOAD_FREQ; - } - - /* - * Flip the idle index... - * - * Make sure we first write the new time then flip the index, so that - * calc_load_write_idx() will see the new time when it reads the new - * index, this avoids a double flip messing things up. - */ - smp_wmb(); - calc_load_idx++; -} -#else /* !CONFIG_NO_HZ_COMMON */ - -static inline long calc_load_fold_idle(void) { return 0; } -static inline void calc_global_nohz(void) { } - -#endif /* CONFIG_NO_HZ_COMMON */ - -/* - * calc_load - update the avenrun load estimates 10 ticks after the - * CPUs have updated calc_load_tasks. - */ -void calc_global_load(unsigned long ticks) -{ - long active, delta; - - if (time_before(jiffies, calc_load_update + 10)) - return; - - /* - * Fold the 'old' idle-delta to include all NO_HZ cpus. - */ - delta = calc_load_fold_idle(); - if (delta) - atomic_long_add(delta, &calc_load_tasks); - - active = atomic_long_read(&calc_load_tasks); - active = active > 0 ? active * FIXED_1 : 0; - - avenrun[0] = calc_load(avenrun[0], EXP_1, active); - avenrun[1] = calc_load(avenrun[1], EXP_5, active); - avenrun[2] = calc_load(avenrun[2], EXP_15, active); - - calc_load_update += LOAD_FREQ; - - /* - * In case we idled for multiple LOAD_FREQ intervals, catch up in bulk. - */ - calc_global_nohz(); -} - -/* - * Called from update_cpu_load() to periodically update this CPU's - * active count. - */ -static void calc_load_account_active(struct rq *this_rq) -{ - long delta; - - if (time_before(jiffies, this_rq->calc_load_update)) - return; - - delta = calc_load_fold_active(this_rq); - if (delta) - atomic_long_add(delta, &calc_load_tasks); - - this_rq->calc_load_update += LOAD_FREQ; -} - -/* - * End of global load-average stuff - */ - -/* - * The exact cpuload at various idx values, calculated at every tick would be - * load = (2^idx - 1) / 2^idx * load + 1 / 2^idx * cur_load - * - * If a cpu misses updates for n-1 ticks (as it was idle) and update gets called - * on nth tick when cpu may be busy, then we have: - * load = ((2^idx - 1) / 2^idx)^(n-1) * load - * load = (2^idx - 1) / 2^idx) * load + 1 / 2^idx * cur_load - * - * decay_load_missed() below does efficient calculation of - * load = ((2^idx - 1) / 2^idx)^(n-1) * load - * avoiding 0..n-1 loop doing load = ((2^idx - 1) / 2^idx) * load - * - * The calculation is approximated on a 128 point scale. - * degrade_zero_ticks is the number of ticks after which load at any - * particular idx is approximated to be zero. - * degrade_factor is a precomputed table, a row for each load idx. - * Each column corresponds to degradation factor for a power of two ticks, - * based on 128 point scale. - * Example: - * row 2, col 3 (=12) says that the degradation at load idx 2 after - * 8 ticks is 12/128 (which is an approximation of exact factor 3^8/4^8). - * - * With this power of 2 load factors, we can degrade the load n times - * by looking at 1 bits in n and doing as many mult/shift instead of - * n mult/shifts needed by the exact degradation. - */ -#define DEGRADE_SHIFT 7 -static const unsigned char - degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128}; -static const unsigned char - degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = { - {0, 0, 0, 0, 0, 0, 0, 0}, - {64, 32, 8, 0, 0, 0, 0, 0}, - {96, 72, 40, 12, 1, 0, 0}, - {112, 98, 75, 43, 15, 1, 0}, - {120, 112, 98, 76, 45, 16, 2} }; - -/* - * Update cpu_load for any missed ticks, due to tickless idle. The backlog - * would be when CPU is idle and so we just decay the old load without - * adding any new load. - */ -static unsigned long -decay_load_missed(unsigned long load, unsigned long missed_updates, int idx) -{ - int j = 0; - - if (!missed_updates) - return load; - - if (missed_updates >= degrade_zero_ticks[idx]) - return 0; - - if (idx == 1) - return load >> missed_updates; - - while (missed_updates) { - if (missed_updates % 2) - load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT; - - missed_updates >>= 1; - j++; - } - return load; -} - -/* - * Update rq->cpu_load[] statistics. This function is usually called every - * scheduler tick (TICK_NSEC). With tickless idle this will not be called - * every tick. We fix it up based on jiffies. - */ -static void __update_cpu_load(struct rq *this_rq, unsigned long this_load, - unsigned long pending_updates) -{ - int i, scale; - - this_rq->nr_load_updates++; - - /* Update our load: */ - this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */ - for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) { - unsigned long old_load, new_load; - - /* scale is effectively 1 << i now, and >> i divides by scale */ - - old_load = this_rq->cpu_load[i]; - old_load = decay_load_missed(old_load, pending_updates - 1, i); - new_load = this_load; - /* - * Round up the averaging division if load is increasing. This - * prevents us from getting stuck on 9 if the load is 10, for - * example. - */ - if (new_load > old_load) - new_load += scale - 1; - - this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i; - } - - sched_avg_update(this_rq); -} - -#ifdef CONFIG_NO_HZ_COMMON -/* - * There is no sane way to deal with nohz on smp when using jiffies because the - * cpu doing the jiffies update might drift wrt the cpu doing the jiffy reading - * causing off-by-one errors in observed deltas; {0,2} instead of {1,1}. - * - * Therefore we cannot use the delta approach from the regular tick since that - * would seriously skew the load calculation. However we'll make do for those - * updates happening while idle (nohz_idle_balance) or coming out of idle - * (tick_nohz_idle_exit). - * - * This means we might still be one tick off for nohz periods. - */ - -/* - * Called from nohz_idle_balance() to update the load ratings before doing the - * idle balance. - */ -void update_idle_cpu_load(struct rq *this_rq) -{ - unsigned long curr_jiffies = ACCESS_ONCE(jiffies); - unsigned long load = this_rq->load.weight; - unsigned long pending_updates; - - /* - * bail if there's load or we're actually up-to-date. - */ - if (load || curr_jiffies == this_rq->last_load_update_tick) - return; - - pending_updates = curr_jiffies - this_rq->last_load_update_tick; - this_rq->last_load_update_tick = curr_jiffies; - - __update_cpu_load(this_rq, load, pending_updates); -} - -/* - * Called from tick_nohz_idle_exit() -- try and fix up the ticks we missed. - */ -void update_cpu_load_nohz(void) -{ - struct rq *this_rq = this_rq(); - unsigned long curr_jiffies = ACCESS_ONCE(jiffies); - unsigned long pending_updates; - - if (curr_jiffies == this_rq->last_load_update_tick) - return; - - raw_spin_lock(&this_rq->lock); - pending_updates = curr_jiffies - this_rq->last_load_update_tick; - if (pending_updates) { - this_rq->last_load_update_tick = curr_jiffies; - /* - * We were idle, this means load 0, the current load might be - * !0 due to remote wakeups and the sort. - */ - __update_cpu_load(this_rq, 0, pending_updates); - } - raw_spin_unlock(&this_rq->lock); -} -#endif /* CONFIG_NO_HZ_COMMON */ - -/* - * Called from scheduler_tick() - */ -static void update_cpu_load_active(struct rq *this_rq) -{ - /* - * See the mess around update_idle_cpu_load() / update_cpu_load_nohz(). - */ - this_rq->last_load_update_tick = jiffies; - __update_cpu_load(this_rq, this_rq->load.weight, 1); - - calc_load_account_active(this_rq); -} - -#if defined(CONFIG_SMP) - -/* - * sched_exec - execve() is a valuable balancing opportunity, because at - * this point the task has the smallest effective memory and cache footprint. - */ -void sched_exec(void) -{ - struct task_struct *p = current; - unsigned long flags; - int dest_cpu; - - if (sched_enable_hmp) - return; - - raw_spin_lock_irqsave(&p->pi_lock, flags); - dest_cpu = p->sched_class->select_task_rq(p, SD_BALANCE_EXEC, 0); - if (dest_cpu == smp_processor_id()) - goto unlock; - - if (likely(cpu_active(dest_cpu))) { - struct migration_arg arg = { p, dest_cpu }; +static void +fire_sched_out_preempt_notifiers(struct task_struct *curr, + struct task_struct *next) +{ + struct preempt_notifier *notifier; - raw_spin_unlock_irqrestore(&p->pi_lock, flags); - stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); - return; - } -unlock: - raw_spin_unlock_irqrestore(&p->pi_lock, flags); + hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) + notifier->ops->sched_out(notifier, next); } -#endif - -DEFINE_PER_CPU(struct kernel_stat, kstat); -DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); - -EXPORT_PER_CPU_SYMBOL(kstat); -EXPORT_PER_CPU_SYMBOL(kernel_cpustat); +#else /* !CONFIG_PREEMPT_NOTIFIERS */ -/* - * Return any ns on the sched_clock that have not yet been accounted in - * @p in case that task is currently running. - * - * Called with task_rq_lock() held on @rq. - */ -static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq) +static void fire_sched_in_preempt_notifiers(struct task_struct *curr) { - u64 ns = 0; - - if (task_current(rq, p)) { - update_rq_clock(rq); - ns = rq->clock_task - p->se.exec_start; - if ((s64)ns < 0) - ns = 0; - } - - return ns; } -unsigned long long task_delta_exec(struct task_struct *p) +static void +fire_sched_out_preempt_notifiers(struct task_struct *curr, + struct task_struct *next) { - unsigned long flags; - struct rq *rq; - u64 ns = 0; - - rq = task_rq_lock(p, &flags); - ns = do_task_delta_exec(p, rq); - task_rq_unlock(rq, p, &flags); - - return ns; } -/* - * Return accounted runtime for the task. - * In case the task is currently running, return the runtime plus current's - * pending runtime that have not been accounted yet. - */ -unsigned long long task_sched_runtime(struct task_struct *p) -{ - unsigned long flags; - struct rq *rq; - u64 ns = 0; - - rq = task_rq_lock(p, &flags); - ns = p->se.sum_exec_runtime + do_task_delta_exec(p, rq); - task_rq_unlock(rq, p, &flags); - - return ns; -} +#endif /* CONFIG_PREEMPT_NOTIFIERS */ -/* - * This function gets called by the timer code, with HZ frequency. - * We call it with interrupts disabled. +/** + * prepare_task_switch - prepare to switch tasks + * @rq: the runqueue preparing to switch + * @prev: the current task that is being switched out + * @next: the task we are going to switch to. + * + * This is called with the rq lock held and interrupts off. It must + * be paired with a subsequent finish_task_switch after the context + * switch. + * + * prepare_task_switch sets up locking and calls architecture specific + * hooks. */ -void scheduler_tick(void) +static inline void +prepare_task_switch(struct rq *rq, struct task_struct *prev, + struct task_struct *next) { - int cpu = smp_processor_id(); - struct rq *rq = cpu_rq(cpu); - struct task_struct *curr = rq->curr; - - sched_clock_tick(); - - raw_spin_lock(&rq->lock); - set_window_start(rq); - update_rq_clock(rq); - update_cpu_load_active(rq); - curr->sched_class->task_tick(rq, curr, 0); - update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), 0); - raw_spin_unlock(&rq->lock); - - perf_event_task_tick(); - -#ifdef CONFIG_SMP - rq->idle_balance = idle_cpu(cpu); - trigger_load_balance(rq, cpu); -#endif - rq_last_tick_reset(rq); - if (curr->sched_class == &fair_sched_class) - check_for_migration(rq, curr); + trace_sched_switch(prev, next); + sched_info_switch(rq, prev, next); + perf_event_task_sched_out(prev, next); + fire_sched_out_preempt_notifiers(prev, next); + prepare_lock_switch(rq, next); + prepare_arch_switch(next); } -#ifdef CONFIG_NO_HZ_FULL /** - * scheduler_tick_max_deferment + * finish_task_switch - clean up after a task-switch + * @rq: runqueue associated with task-switch + * @prev: the thread we just switched away from. * - * Keep at least one tick per second when a single - * active task is running because the scheduler doesn't - * yet completely support full dynticks environment. + * finish_task_switch must be called after the context switch, paired + * with a prepare_task_switch call before the context switch. + * finish_task_switch will reconcile locking set up by prepare_task_switch, + * and do any other architecture-specific cleanup actions. * - * This makes sure that uptime, CFS vruntime, load - * balancing, etc... continue to move forward, even - * with a very low granularity. + * Note that we may have delayed dropping an mm in context_switch(). If + * so, we finish that here outside of the runqueue lock. (Doing it + * with the lock held can cause deadlocks; see schedule() for + * details.) */ -u64 scheduler_tick_max_deferment(void) -{ - struct rq *rq = this_rq(); - unsigned long next, now = ACCESS_ONCE(jiffies); - - next = rq->last_sched_tick + HZ; - - if (time_before_eq(next, now)) - return 0; - - return jiffies_to_usecs(next - now) * NSEC_PER_USEC; -} -#endif - -notrace unsigned long get_parent_ip(unsigned long addr) +static void finish_task_switch(struct rq *rq, struct task_struct *prev) + __releases(rq->lock) { - if (in_lock_functions(addr)) { - addr = CALLER_ADDR2; - if (in_lock_functions(addr)) - addr = CALLER_ADDR3; - } - return addr; -} + struct mm_struct *mm = rq->prev_mm; + long prev_state; -#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \ - defined(CONFIG_PREEMPT_TRACER)) + rq->prev_mm = NULL; -void __kprobes add_preempt_count(int val) -{ -#ifdef CONFIG_DEBUG_PREEMPT - /* - * Underflow? - */ - if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) - return; -#endif - preempt_count() += val; -#ifdef CONFIG_DEBUG_PREEMPT /* - * Spinlock count overflowing soon? + * A task struct has one reference for the use as "current". + * If a task dies, then it sets TASK_DEAD in tsk->state and calls + * schedule one last time. The schedule call will never return, and + * the scheduled task must drop that reference. + * The test for TASK_DEAD must occur while the runqueue locks are + * still held, otherwise prev could be scheduled on another cpu, die + * there before we look at prev->state, and then the reference would + * be dropped twice. + * Manfred Spraul */ - DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= - PREEMPT_MASK - 10); -#endif - if (preempt_count() == val) - trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); -} -EXPORT_SYMBOL(add_preempt_count); + prev_state = prev->state; + vtime_task_switch(prev); + finish_arch_switch(prev); + perf_event_task_sched_in(prev, current); + finish_lock_switch(rq, prev); + finish_arch_post_lock_switch(); -void __kprobes sub_preempt_count(int val) -{ -#ifdef CONFIG_DEBUG_PREEMPT - /* - * Underflow? - */ - if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) - return; - /* - * Is the spinlock portion underflowing? - */ - if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && - !(preempt_count() & PREEMPT_MASK))) - return; -#endif + fire_sched_in_preempt_notifiers(current); + if (mm) + mmdrop(mm); + if (unlikely(prev_state == TASK_DEAD)) { + if (prev->sched_class->task_dead) + prev->sched_class->task_dead(prev); + + /* + * Remove function-return probe instances associated with this + * task and put them back on the free list. + */ + kprobe_flush_task(prev); + put_task_struct(prev); + } - if (preempt_count() == val) - trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); - preempt_count() -= val; + tick_nohz_task_switch(current); } -EXPORT_SYMBOL(sub_preempt_count); -#endif +#ifdef CONFIG_SMP -/* - * Print scheduling while atomic bug: - */ -static noinline void __schedule_bug(struct task_struct *prev) +/* rq->lock is NOT held, but preemption is disabled */ +static inline void post_schedule(struct rq *rq) { - if (oops_in_progress) - return; + if (rq->post_schedule) { + unsigned long flags; - printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", - prev->comm, prev->pid, preempt_count()); + raw_spin_lock_irqsave(&rq->lock, flags); + if (rq->curr->sched_class->post_schedule) + rq->curr->sched_class->post_schedule(rq); + raw_spin_unlock_irqrestore(&rq->lock, flags); - debug_show_held_locks(prev); - print_modules(); - if (irqs_disabled()) - print_irqtrace_events(prev); - dump_stack(); - add_taint(TAINT_WARN, LOCKDEP_STILL_OK); + rq->post_schedule = 0; + } } -/* - * Various schedule()-time debugging checks and statistics: - */ -static inline void schedule_debug(struct task_struct *prev) -{ - /* - * Test if we are atomic. Since do_exit() needs to call into - * schedule() atomically, we ignore that path for now. - * Otherwise, whine if we are scheduling when we should not be. - */ - if (unlikely(in_atomic_preempt_off() && !prev->exit_state)) - __schedule_bug(prev); - rcu_sleep_check(); - - profile_hit(SCHED_PROFILING, __builtin_return_address(0)); - - schedstat_inc(this_rq(), sched_count); -} +#else -static void put_prev_task(struct rq *rq, struct task_struct *prev) +static inline void post_schedule(struct rq *rq) { - if (prev->on_rq || rq->skip_clock_update < 0) - update_rq_clock(rq); - prev->sched_class->put_prev_task(rq, prev); } +>>>>>>> 374fbbf4d13... Backport EAS scheduler from 3.18 -/* - * Pick up the highest-prio task: +#endif + +/** + * schedule_tail - first thing a freshly forked thread must call. + * @prev: the thread we just switched away from. */ -static inline struct task_struct * -pick_next_task(struct rq *rq) +asmlinkage void schedule_tail(struct task_struct *prev) + __releases(rq->lock) { - const struct sched_class *class; - struct task_struct *p; + struct rq *rq = this_rq(); + + finish_task_switch(rq, prev); /* - * Optimization: we know that if all tasks are in - * the fair class we can call that function directly: + * FIXME: do we need to worry about rq being invalidated by the + * task_switch? */ - if (likely(rq->nr_running == rq->cfs.h_nr_running)) { - p = fair_sched_class.pick_next_task(rq); - if (likely(p)) - return p; - } - - for_each_class(class) { - p = class->pick_next_task(rq); - if (p) - return p; - } + post_schedule(rq); - BUG(); /* the idle class will always have a runnable task */ + if (current->set_child_tid) + put_user(task_pid_vnr(current), current->set_child_tid); } /* - * __schedule() is the main scheduler function. - * - * The main means of driving the scheduler and thus entering this function are: - * - * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. - * - * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return - * paths. For example, see arch/x86/entry_64.S. - * - * To drive preemption between tasks, the scheduler sets the flag in timer - * interrupt handler scheduler_tick(). - * - * 3. Wakeups don't really cause entry into schedule(). They add a - * task to the run-queue and that's it. - * - * Now, if the new task added to the run-queue preempts the current - * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets - * called on the nearest possible occasion: - * - * - If the kernel is preemptible (CONFIG_PREEMPT=y): - * - * - in syscall or exception context, at the next outmost - * preempt_enable(). (this might be as soon as the wake_up()'s - * spin_unlock()!) - * - * - in IRQ context, return from interrupt-handler to - * preemptible context - * - * - If the kernel is not preemptible (CONFIG_PREEMPT is not set) - * then at the next: - * - * - cond_resched() call - * - explicit schedule() call - * - return from syscall or exception to user-space - * - return from interrupt-handler to user-space + * context_switch - switch to the new MM and the new + * thread's register state. */ -static void __sched __schedule(void) +static inline void +context_switch(struct rq *rq, struct task_struct *prev, + struct task_struct *next) { - struct task_struct *prev, *next; - unsigned long *switch_count; - struct rq *rq; - int cpu; - u64 wallclock; + struct mm_struct *mm, *oldmm; -need_resched: - preempt_disable(); - cpu = smp_processor_id(); - rq = cpu_rq(cpu); - rcu_note_context_switch(cpu); - prev = rq->curr; + prepare_task_switch(rq, prev, next); - schedule_debug(prev); + mm = next->mm; + oldmm = prev->active_mm; + /* + * For paravirt, this is coupled with an exit in switch_to to + * combine the page table reload and the switch backend into + * one hypercall. + */ + arch_start_context_switch(prev); - if (sched_feat(HRTICK)) - hrtick_clear(rq); + if (!mm) { + next->active_mm = oldmm; + atomic_inc(&oldmm->mm_count); + enter_lazy_tlb(oldmm, next); + } else + switch_mm(oldmm, mm, next); + if (!prev->mm) { + prev->active_mm = NULL; + rq->prev_mm = oldmm; + } /* - * Make sure that signal_pending_state()->signal_pending() below - * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) - * done by the caller to avoid the race with signal_wake_up(). + * Since the runqueue lock will be released by the next + * task (which is an invalid locking op but in the case + * of the scheduler it's an obvious special-case), so we + * do an early lockdep release here: */ - smp_mb__before_spinlock(); - raw_spin_lock_irq(&rq->lock); + spin_release(&rq->lock.dep_map, 1, _THIS_IP_); -#ifdef CONFIG_ARCH_WANTS_CTXSW_LOGGING - dlog("%s: locked %p at %llu\n", __func__, &rq->lock, sched_clock()); -#endif + context_tracking_task_switch(prev, next); + /* Here we just switch the register state and the stack. */ + switch_to(prev, next, prev); - switch_count = &prev->nivcsw; - if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { - if (unlikely(signal_pending_state(prev->state, prev))) { - prev->state = TASK_RUNNING; - } else { - deactivate_task(rq, prev, DEQUEUE_SLEEP); - prev->on_rq = 0; + barrier(); + /* + * this_rq must be evaluated again because prev may have moved + * CPUs since it called schedule(), thus the 'rq' on its stack + * frame will be invalid. + */ + finish_task_switch(this_rq(), prev); +} - /* - * If a worker went to sleep, notify and ask workqueue - * whether it wants to wake up a task to maintain - * concurrency. - */ - if (prev->flags & PF_WQ_WORKER) { - struct task_struct *to_wakeup; +/* + * nr_running and nr_context_switches: + * + * externally visible scheduler statistics: current number of runnable + * threads, total number of context switches performed since bootup. + */ +unsigned long nr_running(void) +{ + unsigned long i, sum = 0; - to_wakeup = wq_worker_sleeping(prev, cpu); - if (to_wakeup) - try_to_wake_up_local(to_wakeup); - } - } - switch_count = &prev->nvcsw; - } + for_each_online_cpu(i) + sum += cpu_rq(i)->nr_running; - pre_schedule(rq, prev); + return sum; +} - if (unlikely(!rq->nr_running)) - idle_balance(cpu, rq); +/* + * Check if only the current task is running on the cpu. + */ +bool single_task_running(void) +{ + if (cpu_rq(smp_processor_id())->nr_running == 1) + return true; + else + return false; +} +EXPORT_SYMBOL(single_task_running); - put_prev_task(rq, prev); - next = pick_next_task(rq); - wallclock = sched_clock(); - update_task_ravg(prev, rq, PUT_PREV_TASK, wallclock, 0); - update_task_ravg(next, rq, PICK_NEXT_TASK, wallclock, 0); - clear_tsk_need_resched(prev); - rq->skip_clock_update = 0; +unsigned long long nr_context_switches(void) +{ + int i; + unsigned long long sum = 0; - BUG_ON(task_cpu(next) != cpu_of(rq)); + for_each_possible_cpu(i) + sum += cpu_rq(i)->nr_switches; - if (likely(prev != next)) { - rq->nr_switches++; - rq->curr = next; - ++*switch_count; + return sum; +} -#ifdef CONFIG_ARCH_WANTS_CTXSW_LOGGING - dlog("%s: enter context_switch at %llu\n", - __func__, sched_clock()); -#endif - context_switch(rq, prev, next); /* unlocks the rq */ - /* - * The context switch have flipped the stack from under us - * and restored the local variables which were saved when - * this task called schedule() in the past. prev == current - * is still correct, but it can be moved to another cpu/rq. - */ - cpu = smp_processor_id(); - rq = cpu_rq(cpu); - } else - raw_spin_unlock_irq(&rq->lock); +unsigned long nr_iowait(void) +{ + unsigned long i, sum = 0; - post_schedule(rq); + for_each_possible_cpu(i) + sum += atomic_read(&cpu_rq(i)->nr_iowait); - sched_preempt_enable_no_resched(); - if (need_resched()) - goto need_resched; + return sum; } -static inline void sched_submit_work(struct task_struct *tsk) +unsigned long nr_iowait_cpu(int cpu) { - if (!tsk->state || tsk_is_pi_blocked(tsk)) - return; - /* - * If we are going to sleep and we have plugged IO queued, - * make sure to submit it to avoid deadlocks. - */ - if (blk_needs_flush_plug(tsk)) - blk_schedule_flush_plug(tsk); + struct rq *this = cpu_rq(cpu); + return atomic_read(&this->nr_iowait); } -asmlinkage void __sched schedule(void) +void get_iowait_load(unsigned long *nr_waiters, unsigned long *load) { - struct task_struct *tsk = current; - - sched_submit_work(tsk); - __schedule(); + struct rq *rq = this_rq(); + *nr_waiters = atomic_read(&rq->nr_iowait); + *load = rq->load.weight; } -EXPORT_SYMBOL(schedule); -#ifdef CONFIG_CONTEXT_TRACKING -asmlinkage void __sched schedule_user(void) +#ifdef CONFIG_SMP + +/* + * sched_exec - execve() is a valuable balancing opportunity, because at + * this point the task has the smallest effective memory and cache footprint. + */ +void sched_exec(void) { - /* - * If we come here after a random call to set_need_resched(), - * or we have been woken up remotely but the IPI has not yet arrived, - * we haven't yet exited the RCU idle mode. Do it here manually until - * we find a better solution. - */ - user_exit(); - schedule(); - user_enter(); + struct task_struct *p = current; + unsigned long flags; + int dest_cpu; + + raw_spin_lock_irqsave(&p->pi_lock, flags); + dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0); + if (dest_cpu == smp_processor_id()) + goto unlock; + + if (likely(cpu_active(dest_cpu))) { + struct migration_arg arg = { p, dest_cpu }; + + raw_spin_unlock_irqrestore(&p->pi_lock, flags); + stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); + return; + } +unlock: + raw_spin_unlock_irqrestore(&p->pi_lock, flags); } + #endif -/** - * schedule_preempt_disabled - called with preemption disabled +DEFINE_PER_CPU(struct kernel_stat, kstat); +DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); + +EXPORT_PER_CPU_SYMBOL(kstat); +EXPORT_PER_CPU_SYMBOL(kernel_cpustat); + +/* + * Return any ns on the sched_clock that have not yet been accounted in + * @p in case that task is currently running. * - * Returns with preemption disabled. Note: preempt_count must be 1 + * Called with task_rq_lock() held on @rq. */ -void __sched schedule_preempt_disabled(void) +static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq) { - sched_preempt_enable_no_resched(); - schedule(); - preempt_disable(); + u64 ns = 0; + + if (task_current(rq, p)) { + update_rq_clock(rq); + ns = rq->clock_task - p->se.exec_start; + if ((s64)ns < 0) + ns = 0; + } + + return ns; +} + +unsigned long long task_delta_exec(struct task_struct *p) +{ + unsigned long flags; + struct rq *rq; + u64 ns = 0; + + rq = task_rq_lock(p, &flags); + ns = do_task_delta_exec(p, rq); + task_rq_unlock(rq, p, &flags); + + return ns; } -#ifdef CONFIG_PREEMPT /* - * this is the entry point to schedule() from in-kernel preemption - * off of preempt_enable. Kernel preemptions off return from interrupt - * occur there and call schedule directly. + * Return accounted runtime for the task. + * In case the task is currently running, return the runtime plus current's + * pending runtime that have not been accounted yet. */ -asmlinkage void __sched notrace preempt_schedule(void) + +unsigned long long task_sched_runtime(struct task_struct *p) { - struct thread_info *ti = current_thread_info(); + unsigned long flags; + struct rq *rq; + u64 ns; +#if defined(CONFIG_64BIT) && defined(CONFIG_SMP) /* - * If there is a non-zero preempt_count or interrupts are disabled, - * we do not want to preempt the current task. Just return.. + * 64-bit doesn't need locks to atomically read a 64bit value. + * So we have a optimization chance when the task's delta_exec is 0. + * Reading ->on_cpu is racy, but this is ok. + * + * If we race with it leaving cpu, we'll take a lock. So we're correct. + * If we race with it entering cpu, unaccounted time is 0. This is + * indistinguishable from the read occurring a few cycles earlier. + * If we see ->on_cpu without ->on_rq, the task is leaving, and has + * been accounted, so we're correct here as well. */ - if (likely(ti->preempt_count || irqs_disabled())) - return; + if (!p->on_cpu || !task_on_rq_queued(p)) + return p->se.sum_exec_runtime; +#endif - do { - add_preempt_count_notrace(PREEMPT_ACTIVE); - __schedule(); - sub_preempt_count_notrace(PREEMPT_ACTIVE); + rq = task_rq_lock(p, &flags); + /* + * Must be ->curr _and_ ->on_rq. If dequeued, we would + * project cycles that may never be accounted to this + * thread, breaking clock_gettime(). + */ + if (task_current(rq, p) && task_on_rq_queued(p)) { + update_rq_clock(rq); + p->sched_class->update_curr(rq); + } + ns = p->se.sum_exec_runtime; + task_rq_unlock(rq, p, &flags); - /* - * Check again in case we missed a preemption opportunity - * between schedule and now. - */ - barrier(); - } while (need_resched()); + return ns; } -EXPORT_SYMBOL(preempt_schedule); /* - * this is the entry point to schedule() from kernel preemption - * off of irq context. - * Note, that this is called and return with irqs disabled. This will - * protect us against recursive calling from irq. + * This function gets called by the timer code, with HZ frequency. + * We call it with interrupts disabled. */ -asmlinkage void __sched preempt_schedule_irq(void) +void scheduler_tick(void) { - struct thread_info *ti = current_thread_info(); - enum ctx_state prev_state; - - /* Catch callers which need to be fixed */ - BUG_ON(ti->preempt_count || !irqs_disabled()); - - prev_state = exception_enter(); - - do { - add_preempt_count(PREEMPT_ACTIVE); - local_irq_enable(); - __schedule(); - local_irq_disable(); - sub_preempt_count(PREEMPT_ACTIVE); - - /* - * Check again in case we missed a preemption opportunity - * between schedule and now. - */ - barrier(); - } while (need_resched()); + int cpu = smp_processor_id(); + struct rq *rq = cpu_rq(cpu); + struct task_struct *curr = rq->curr; - exception_exit(prev_state); -} + sched_clock_tick(); -#endif /* CONFIG_PREEMPT */ + raw_spin_lock(&rq->lock); + update_rq_clock(rq); + curr->sched_class->task_tick(rq, curr, 0); + update_cpu_load_active(rq); + calc_global_load_tick(rq); + raw_spin_unlock(&rq->lock); -int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags, - void *key) -{ - return try_to_wake_up(curr->private, mode, wake_flags); + perf_event_task_tick(); +#ifdef CONFIG_MT_SCHED_MONITOR + if (smp_processor_id() == 0) /* only record by CPU#0 */ + mt_save_irq_counts(); +#endif +#ifdef CONFIG_SMP + rq->idle_balance = idle_cpu(cpu); + trigger_load_balance(rq); +#endif + rq_last_tick_reset(rq); } -EXPORT_SYMBOL(default_wake_function); -/* - * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just - * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve - * number) then we wake all the non-exclusive tasks and one exclusive task. +#ifdef CONFIG_NO_HZ_FULL +/** + * scheduler_tick_max_deferment * - * There are circumstances in which we can try to wake a task which has already - * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns - * zero in this (rare) case, and we handle it by continuing to scan the queue. + * Keep at least one tick per second when a single + * active task is running because the scheduler doesn't + * yet completely support full dynticks environment. + * + * This makes sure that uptime, CFS vruntime, load + * balancing, etc... continue to move forward, even + * with a very low granularity. + * + * Return: Maximum deferment in nanoseconds. */ -static void __wake_up_common(wait_queue_head_t *q, unsigned int mode, - int nr_exclusive, int wake_flags, void *key) +u64 scheduler_tick_max_deferment(void) { - wait_queue_t *curr, *next; + struct rq *rq = this_rq(); + unsigned long next, now = ACCESS_ONCE(jiffies); - list_for_each_entry_safe(curr, next, &q->task_list, task_list) { - unsigned flags = curr->flags; + next = rq->last_sched_tick + HZ; - if (curr->func(curr, mode, wake_flags, key) && - (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive) - break; - } + if (time_before_eq(next, now)) + return 0; + + return jiffies_to_nsecs(next - now); } +#endif -/** - * __wake_up - wake up threads blocked on a waitqueue. - * @q: the waitqueue - * @mode: which threads - * @nr_exclusive: how many wake-one or wake-many threads to wake up - * @key: is directly passed to the wakeup function - * - * It may be assumed that this function implies a write memory barrier before - * changing the task state if and only if any tasks are woken up. - */ -void __wake_up(wait_queue_head_t *q, unsigned int mode, - int nr_exclusive, void *key) +notrace unsigned long get_parent_ip(unsigned long addr) { - unsigned long flags; + if (in_lock_functions(addr)) { + addr = CALLER_ADDR2; + if (in_lock_functions(addr)) + addr = CALLER_ADDR3; + } + return addr; +} - spin_lock_irqsave(&q->lock, flags); - __wake_up_common(q, mode, nr_exclusive, 0, key); - spin_unlock_irqrestore(&q->lock, flags); +#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \ + defined(CONFIG_PREEMPT_TRACER)) + +void preempt_count_add(int val) +{ +#ifdef CONFIG_DEBUG_PREEMPT + /* + * Underflow? + */ + if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) + return; +#endif + preempt_count() += val; +#ifdef CONFIG_DEBUG_PREEMPT + /* + * Spinlock count overflowing soon? + */ + DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= + PREEMPT_MASK - 10); +#endif + if (preempt_count() == val) { + unsigned long ip = get_parent_ip(CALLER_ADDR1); +#ifdef CONFIG_DEBUG_PREEMPT + current->preempt_disable_ip = ip; +#endif + trace_preempt_off(CALLER_ADDR0, ip); +#if defined(CONFIG_PREEMPT_MONITOR) && defined(CONFIG_MTPROF) + if (unlikely(__raw_get_cpu_var(mtsched_mon_enabled) & 0x1)) { + /* current->t_add_prmpt = sched_clock(); */ + MT_trace_preempt_off(); + } +#endif + } } -EXPORT_SYMBOL(__wake_up); +EXPORT_SYMBOL(preempt_count_add); + +#endif /* - * Same as __wake_up but called with the spinlock in wait_queue_head_t held. + * Print scheduling while atomic bug: */ -void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr) +static noinline void __schedule_bug(struct task_struct *prev) { - __wake_up_common(q, mode, nr, 0, NULL); -} -EXPORT_SYMBOL_GPL(__wake_up_locked); + if (oops_in_progress) + return; -void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key) -{ - __wake_up_common(q, mode, 1, 0, key); -} -EXPORT_SYMBOL_GPL(__wake_up_locked_key); + printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", + prev->comm, prev->pid, preempt_count()); -/** - * __wake_up_sync_key - wake up threads blocked on a waitqueue. - * @q: the waitqueue - * @mode: which threads - * @nr_exclusive: how many wake-one or wake-many threads to wake up - * @key: opaque value to be passed to wakeup targets - * - * The sync wakeup differs that the waker knows that it will schedule - * away soon, so while the target thread will be woken up, it will not - * be migrated to another CPU - ie. the two threads are 'synchronized' - * with each other. This can prevent needless bouncing between CPUs. - * - * On UP it can prevent extra preemption. - * - * It may be assumed that this function implies a write memory barrier before - * changing the task state if and only if any tasks are woken up. + debug_show_held_locks(prev); + print_modules(); + if (irqs_disabled()) + print_irqtrace_events(prev); +#ifdef CONFIG_DEBUG_PREEMPT + if (in_atomic_preempt_off()) { + pr_err("Preemption disabled at:"); + print_ip_sym(current->preempt_disable_ip); + pr_cont("\n"); + } +#endif + dump_stack(); + add_taint(TAINT_WARN, LOCKDEP_STILL_OK); +} + +/* + * Various schedule()-time debugging checks and statistics: */ -void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, - int nr_exclusive, void *key) +static inline void schedule_debug(struct task_struct *prev) { - unsigned long flags; - int wake_flags = WF_SYNC; - - if (unlikely(!q)) - return; +#ifdef CONFIG_SCHED_STACK_END_CHECK + BUG_ON(unlikely(task_stack_end_corrupted(prev))); +#endif + /* + * Test if we are atomic. Since do_exit() needs to call into + * schedule() atomically, we ignore that path. Otherwise whine + * if we are scheduling when we should not. + */ + if (unlikely(in_atomic_preempt_off() && prev->state != TASK_DEAD)) + __schedule_bug(prev); + rcu_sleep_check(); - if (unlikely(!nr_exclusive)) - wake_flags = 0; + profile_hit(SCHED_PROFILING, __builtin_return_address(0)); - spin_lock_irqsave(&q->lock, flags); - __wake_up_common(q, mode, nr_exclusive, wake_flags, key); - spin_unlock_irqrestore(&q->lock, flags); + schedstat_inc(this_rq(), sched_count); } -EXPORT_SYMBOL_GPL(__wake_up_sync_key); /* - * __wake_up_sync - see __wake_up_sync_key() + * Pick up the highest-prio task: */ -void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive) +static inline struct task_struct * +pick_next_task(struct rq *rq, struct task_struct *prev) { - __wake_up_sync_key(q, mode, nr_exclusive, NULL); + const struct sched_class *class = &fair_sched_class; + struct task_struct *p; + + /* + * Optimization: we know that if all tasks are in + * the fair class we can call that function directly: + */ + if (likely(prev->sched_class == class && + rq->nr_running == rq->cfs.h_nr_running)) { + p = fair_sched_class.pick_next_task(rq, prev); + if (unlikely(p == RETRY_TASK)) + goto again; + + /* assumes fair_sched_class->next == idle_sched_class */ + if (unlikely(!p)) + p = idle_sched_class.pick_next_task(rq, prev); + + return p; + } + +again: + for_each_class(class) { + p = class->pick_next_task(rq, prev); + if (p) { + if (unlikely(p == RETRY_TASK)) + goto again; + return p; + } + } + + BUG(); /* the idle class will always have a runnable task */ } -EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ -/** - * complete: - signals a single thread waiting on this completion - * @x: holds the state of this particular completion +/* + * __schedule() is the main scheduler function. * - * This will wake up a single thread waiting on this completion. Threads will be - * awakened in the same order in which they were queued. + * The main means of driving the scheduler and thus entering this function are: * - * See also complete_all(), wait_for_completion() and related routines. + * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. * - * It may be assumed that this function implies a write memory barrier before - * changing the task state if and only if any tasks are woken up. - */ -void complete(struct completion *x) -{ - unsigned long flags; - - spin_lock_irqsave(&x->wait.lock, flags); - x->done++; - __wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL); - spin_unlock_irqrestore(&x->wait.lock, flags); -} -EXPORT_SYMBOL(complete); - -/** - * complete_all: - signals all threads waiting on this completion - * @x: holds the state of this particular completion + * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return + * paths. For example, see arch/x86/entry_64.S. * - * This will wake up all threads waiting on this particular completion event. + * To drive preemption between tasks, the scheduler sets the flag in timer + * interrupt handler scheduler_tick(). * - * It may be assumed that this function implies a write memory barrier before - * changing the task state if and only if any tasks are woken up. + * 3. Wakeups don't really cause entry into schedule(). They add a + * task to the run-queue and that's it. + * + * Now, if the new task added to the run-queue preempts the current + * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets + * called on the nearest possible occasion: + * + * - If the kernel is preemptible (CONFIG_PREEMPT=y): + * + * - in syscall or exception context, at the next outmost + * preempt_enable(). (this might be as soon as the wake_up()'s + * spin_unlock()!) + * + * - in IRQ context, return from interrupt-handler to + * preemptible context + * + * - If the kernel is not preemptible (CONFIG_PREEMPT is not set) + * then at the next: + * + * - cond_resched() call + * - explicit schedule() call + * - return from syscall or exception to user-space + * - return from interrupt-handler to user-space */ -void complete_all(struct completion *x) +static void __sched __schedule(void) { - unsigned long flags; + struct task_struct *prev, *next; + unsigned long *switch_count; + struct rq *rq; + int cpu; - spin_lock_irqsave(&x->wait.lock, flags); - x->done += UINT_MAX/2; - __wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL); - spin_unlock_irqrestore(&x->wait.lock, flags); -} -EXPORT_SYMBOL(complete_all); +need_resched: + preempt_disable(); + cpu = smp_processor_id(); + rq = cpu_rq(cpu); + rcu_note_context_switch(cpu); + prev = rq->curr; -static inline long __sched -do_wait_for_common(struct completion *x, - long (*action)(long), long timeout, int state) -{ - if (!x->done) { - DECLARE_WAITQUEUE(wait, current); + schedule_debug(prev); - __add_wait_queue_tail_exclusive(&x->wait, &wait); - do { - if (signal_pending_state(state, current)) { - timeout = -ERESTARTSYS; - break; + if (sched_feat(HRTICK)) + hrtick_clear(rq); +#if defined(CONFIG_MT_SCHED_MONITOR) && defined(CONFIG_MTPROF) + __raw_get_cpu_var(MT_trace_in_sched) = 1; +#endif + + /* + * Make sure that signal_pending_state()->signal_pending() below + * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) + * done by the caller to avoid the race with signal_wake_up(). + */ + smp_mb__before_spinlock(); + raw_spin_lock_irq(&rq->lock); + + switch_count = &prev->nivcsw; + if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { + if (unlikely(signal_pending_state(prev->state, prev))) { + prev->state = TASK_RUNNING; + } else { + deactivate_task(rq, prev, DEQUEUE_SLEEP); + prev->on_rq = 0; + + /* + * If a worker went to sleep, notify and ask workqueue + * whether it wants to wake up a task to maintain + * concurrency. + */ + if (prev->flags & PF_WQ_WORKER) { + struct task_struct *to_wakeup; + + to_wakeup = wq_worker_sleeping(prev, cpu); + if (to_wakeup) + try_to_wake_up_local(to_wakeup); } - __set_current_state(state); - spin_unlock_irq(&x->wait.lock); - timeout = action(timeout); - spin_lock_irq(&x->wait.lock); - } while (!x->done && timeout); - __remove_wait_queue(&x->wait, &wait); - if (!x->done) - return timeout; + } + switch_count = &prev->nvcsw; } - x->done--; - return timeout ?: 1; -} - -static inline long __sched -__wait_for_common(struct completion *x, - long (*action)(long), long timeout, int state) -{ - might_sleep(); - spin_lock_irq(&x->wait.lock); - timeout = do_wait_for_common(x, action, timeout, state); - spin_unlock_irq(&x->wait.lock); - return timeout; -} + if (task_on_rq_queued(prev) || rq->skip_clock_update < 0) + update_rq_clock(rq); -static long __sched -wait_for_common(struct completion *x, long timeout, int state) -{ - return __wait_for_common(x, schedule_timeout, timeout, state); -} + next = pick_next_task(rq, prev); + clear_tsk_need_resched(prev); + //TJK clear_preempt_need_resched(); + rq->skip_clock_update = 0; -static long __sched -wait_for_common_io(struct completion *x, long timeout, int state) -{ - return __wait_for_common(x, io_schedule_timeout, timeout, state); -} + if (likely(prev != next)) { + rq->nr_switches++; + rq->curr = next; + ++*switch_count; -/** - * wait_for_completion: - waits for completion of a task - * @x: holds the state of this particular completion - * - * This waits to be signaled for completion of a specific task. It is NOT - * interruptible and there is no timeout. - * - * See also similar routines (i.e. wait_for_completion_timeout()) with timeout - * and interrupt capability. Also see complete(). - */ -void __sched wait_for_completion(struct completion *x) -{ - wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); -} -EXPORT_SYMBOL(wait_for_completion); + context_switch(rq, prev, next); /* unlocks the rq */ + /* + * The context switch have flipped the stack from under us + * and restored the local variables which were saved when + * this task called schedule() in the past. prev == current + * is still correct, but it can be moved to another cpu/rq. + */ + cpu = smp_processor_id(); + rq = cpu_rq(cpu); + } else + raw_spin_unlock_irq(&rq->lock); -/** - * wait_for_completion_timeout: - waits for completion of a task (w/timeout) - * @x: holds the state of this particular completion - * @timeout: timeout value in jiffies - * - * This waits for either a completion of a specific task to be signaled or for a - * specified timeout to expire. The timeout is in jiffies. It is not - * interruptible. - * - * The return value is 0 if timed out, and positive (at least 1, or number of - * jiffies left till timeout) if completed. - */ -unsigned long __sched -wait_for_completion_timeout(struct completion *x, unsigned long timeout) -{ - return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE); -} -EXPORT_SYMBOL(wait_for_completion_timeout); +#if defined(CONFIG_MT_SCHED_MONITOR) && defined(CONFIG_MTPROF) + __raw_get_cpu_var(MT_trace_in_sched) = 0; +#endif + post_schedule(rq); -/** - * wait_for_completion_io: - waits for completion of a task - * @x: holds the state of this particular completion - * - * This waits to be signaled for completion of a specific task. It is NOT - * interruptible and there is no timeout. The caller is accounted as waiting - * for IO. - */ -void __sched wait_for_completion_io(struct completion *x) -{ - wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); + sched_preempt_enable_no_resched(); + if (need_resched()) + goto need_resched; } -EXPORT_SYMBOL(wait_for_completion_io); -/** - * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout) - * @x: holds the state of this particular completion - * @timeout: timeout value in jiffies - * - * This waits for either a completion of a specific task to be signaled or for a - * specified timeout to expire. The timeout is in jiffies. It is not - * interruptible. The caller is accounted as waiting for IO. - * - * The return value is 0 if timed out, and positive (at least 1, or number of - * jiffies left till timeout) if completed. - */ -unsigned long __sched -wait_for_completion_io_timeout(struct completion *x, unsigned long timeout) +static inline void sched_submit_work(struct task_struct *tsk) { - return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE); + if (!tsk->state || tsk_is_pi_blocked(tsk)) + return; + /* + * If we are going to sleep and we have plugged IO queued, + * make sure to submit it to avoid deadlocks. + */ + if (blk_needs_flush_plug(tsk)) + blk_schedule_flush_plug(tsk); } -EXPORT_SYMBOL(wait_for_completion_io_timeout); -/** - * wait_for_completion_interruptible: - waits for completion of a task (w/intr) - * @x: holds the state of this particular completion - * - * This waits for completion of a specific task to be signaled. It is - * interruptible. - * - * The return value is -ERESTARTSYS if interrupted, 0 if completed. - */ -int __sched wait_for_completion_interruptible(struct completion *x) +asmlinkage void __sched schedule(void) { - long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE); - if (t == -ERESTARTSYS) - return t; - return 0; -} -EXPORT_SYMBOL(wait_for_completion_interruptible); + struct task_struct *tsk = current; -/** - * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr)) - * @x: holds the state of this particular completion - * @timeout: timeout value in jiffies - * - * This waits for either a completion of a specific task to be signaled or for a - * specified timeout to expire. It is interruptible. The timeout is in jiffies. - * - * The return value is -ERESTARTSYS if interrupted, 0 if timed out, - * positive (at least 1, or number of jiffies left till timeout) if completed. - */ -long __sched -wait_for_completion_interruptible_timeout(struct completion *x, - unsigned long timeout) -{ - return wait_for_common(x, timeout, TASK_INTERRUPTIBLE); + sched_submit_work(tsk); + __schedule(); } -EXPORT_SYMBOL(wait_for_completion_interruptible_timeout); +EXPORT_SYMBOL(schedule); -/** - * wait_for_completion_killable: - waits for completion of a task (killable) - * @x: holds the state of this particular completion - * - * This waits to be signaled for completion of a specific task. It can be - * interrupted by a kill signal. - * - * The return value is -ERESTARTSYS if interrupted, 0 if completed. - */ -int __sched wait_for_completion_killable(struct completion *x) +#ifdef CONFIG_CONTEXT_TRACKING +asmlinkage void __sched schedule_user(void) { - long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE); - if (t == -ERESTARTSYS) - return t; - return 0; + /* + * If we come here after a random call to set_need_resched(), + * or we have been woken up remotely but the IPI has not yet arrived, + * we haven't yet exited the RCU idle mode. Do it here manually until + * we find a better solution. + * + * NB: There are buggy callers of this function. Ideally we + * should warn if prev_state != IN_USER, but that will trigger + * too frequently to make sense yet. + */ + enum ctx_state prev_state = exception_enter(); + schedule(); + exception_exit(prev_state); } -EXPORT_SYMBOL(wait_for_completion_killable); +#endif /** - * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable)) - * @x: holds the state of this particular completion - * @timeout: timeout value in jiffies - * - * This waits for either a completion of a specific task to be - * signaled or for a specified timeout to expire. It can be - * interrupted by a kill signal. The timeout is in jiffies. + * schedule_preempt_disabled - called with preemption disabled * - * The return value is -ERESTARTSYS if interrupted, 0 if timed out, - * positive (at least 1, or number of jiffies left till timeout) if completed. + * Returns with preemption disabled. Note: preempt_count must be 1 */ -long __sched -wait_for_completion_killable_timeout(struct completion *x, - unsigned long timeout) +void __sched schedule_preempt_disabled(void) { - return wait_for_common(x, timeout, TASK_KILLABLE); + sched_preempt_enable_no_resched(); + schedule(); + preempt_disable(); } -EXPORT_SYMBOL(wait_for_completion_killable_timeout); -/** - * try_wait_for_completion - try to decrement a completion without blocking - * @x: completion structure - * - * Returns: 0 if a decrement cannot be done without blocking - * 1 if a decrement succeeded. - * - * If a completion is being used as a counting completion, - * attempt to decrement the counter without blocking. This - * enables us to avoid waiting if the resource the completion - * is protecting is not available. +#ifdef CONFIG_PREEMPT +/* + * this is the entry point to schedule() from in-kernel preemption + * off of preempt_enable. Kernel preemptions off return from interrupt + * occur there and call schedule directly. */ -bool try_wait_for_completion(struct completion *x) +asmlinkage void __sched notrace preempt_schedule(void) { - unsigned long flags; - int ret = 1; + struct thread_info *ti = current_thread_info(); - spin_lock_irqsave(&x->wait.lock, flags); - if (!x->done) - ret = 0; - else - x->done--; - spin_unlock_irqrestore(&x->wait.lock, flags); - return ret; + /* + * If there is a non-zero preempt_count or interrupts are disabled, + * we do not want to preempt the current task. Just return.. + */ + if (likely(ti->preempt_count || irqs_disabled())) + return; + + do { + add_preempt_count_notrace(PREEMPT_ACTIVE); + __schedule(); + sub_preempt_count_notrace(PREEMPT_ACTIVE); + + /* + * Check again in case we missed a preemption opportunity + * between schedule and now. + */ + barrier(); + } while (need_resched()); } -EXPORT_SYMBOL(try_wait_for_completion); +//NOKPROBE_SYMBOL(preempt_schedule); +EXPORT_SYMBOL(preempt_schedule); +//#ifdef CONFIG_CONTEXT_TRACKING /** - * completion_done - Test to see if a completion has any waiters - * @x: completion structure + * preempt_schedule_context - preempt_schedule called by tracing * - * Returns: 0 if there are waiters (wait_for_completion() in progress) - * 1 if there are no waiters. + * The tracing infrastructure uses preempt_enable_notrace to prevent + * recursion and tracing preempt enabling caused by the tracing + * infrastructure itself. But as tracing can happen in areas coming + * from userspace or just about to enter userspace, a preempt enable + * can occur before user_exit() is called. This will cause the scheduler + * to be called when the system is still in usermode. * + * To prevent this, the preempt_enable_notrace will use this function + * instead of preempt_schedule() to exit user context if needed before + * calling the scheduler. */ -bool completion_done(struct completion *x) +asmlinkage void __sched notrace preempt_schedule_context(void) { - unsigned long flags; - int ret = 1; + enum ctx_state prev_ctx; - spin_lock_irqsave(&x->wait.lock, flags); - if (!x->done) - ret = 0; - spin_unlock_irqrestore(&x->wait.lock, flags); - return ret; -} -EXPORT_SYMBOL(completion_done); + if (likely(!preemptible())) + return; -static long __sched -sleep_on_common(wait_queue_head_t *q, int state, long timeout) -{ - unsigned long flags; - wait_queue_t wait; + do { + add_preempt_count(PREEMPT_ACTIVE); + /* + * Needs preempt disabled in case user_exit() is traced + * and the tracer calls preempt_enable_notrace() causing + * an infinite recursion. + */ + prev_ctx = exception_enter(); + __schedule(); + exception_exit(prev_ctx); - init_waitqueue_entry(&wait, current); + sub_preempt_count(PREEMPT_ACTIVE); + barrier(); + } while (need_resched()); +} +EXPORT_SYMBOL_GPL(preempt_schedule_context); +//#endif /* CONFIG_CONTEXT_TRACKING */ - __set_current_state(state); +#endif /* CONFIG_PREEMPT */ - spin_lock_irqsave(&q->lock, flags); - __add_wait_queue(q, &wait); - spin_unlock(&q->lock); - timeout = schedule_timeout(timeout); - spin_lock_irq(&q->lock); - __remove_wait_queue(q, &wait); - spin_unlock_irqrestore(&q->lock, flags); +/* + * this is the entry point to schedule() from kernel preemption + * off of irq context. + * Note, that this is called and return with irqs disabled. This will + * protect us against recursive calling from irq. + */ +asmlinkage void __sched preempt_schedule_irq(void) +{ + struct thread_info *ti = current_thread_info(); + enum ctx_state prev_state; - return timeout; -} + /* Catch callers which need to be fixed */ + BUG_ON(ti->preempt_count || !irqs_disabled()); -void __sched interruptible_sleep_on(wait_queue_head_t *q) -{ - sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); -} -EXPORT_SYMBOL(interruptible_sleep_on); + prev_state = exception_enter(); -long __sched -interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout) -{ - return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout); -} -EXPORT_SYMBOL(interruptible_sleep_on_timeout); + do { + add_preempt_count(PREEMPT_ACTIVE); + local_irq_enable(); + __schedule(); + local_irq_disable(); + sub_preempt_count(PREEMPT_ACTIVE); -void __sched sleep_on(wait_queue_head_t *q) -{ - sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); + /* + * Check again in case we missed a preemption opportunity + * between schedule and now. + */ + barrier(); + } while (need_resched()); + + exception_exit(prev_state); } -EXPORT_SYMBOL(sleep_on); -long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout) +int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags, + void *key) { - return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout); + return try_to_wake_up(curr->private, mode, wake_flags); } -EXPORT_SYMBOL(sleep_on_timeout); +EXPORT_SYMBOL(default_wake_function); #ifdef CONFIG_RT_MUTEXES @@ -5505,15 +3265,16 @@ EXPORT_SYMBOL(sleep_on_timeout); * This function changes the 'effective' priority of a task. It does * not touch ->normal_prio like __setscheduler(). * - * Used by the rt_mutex code to implement priority inheritance logic. + * Used by the rt_mutex code to implement priority inheritance + * logic. Call site only calls if the priority of the task changed. */ void rt_mutex_setprio(struct task_struct *p, int prio) { - int oldprio, on_rq, running; + int oldprio, queued, running, enqueue_flag = 0; struct rq *rq; const struct sched_class *prev_class; - BUG_ON(prio < 0 || prio > MAX_PRIO); + BUG_ON(prio > MAX_PRIO); rq = __task_rq_lock(p); @@ -5538,24 +3299,52 @@ void rt_mutex_setprio(struct task_struct *p, int prio) trace_sched_pi_setprio(p, prio); oldprio = p->prio; prev_class = p->sched_class; - on_rq = p->on_rq; + queued = task_on_rq_queued(p); running = task_current(rq, p); - if (on_rq) + if (queued) dequeue_task(rq, p, 0); if (running) - p->sched_class->put_prev_task(rq, p); + put_prev_task(rq, p); - if (rt_prio(prio)) + /* + * Boosting condition are: + * 1. -rt task is running and holds mutex A + * --> -dl task blocks on mutex A + * + * 2. -dl task is running and holds mutex A + * --> -dl task blocks on mutex A and could preempt the + * running task + */ + if (dl_prio(prio)) { + struct task_struct *pi_task = rt_mutex_get_top_task(p); + if (!dl_prio(p->normal_prio) || + (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) { + p->dl.dl_boosted = 1; + p->dl.dl_throttled = 0; + enqueue_flag = ENQUEUE_REPLENISH; + } else + p->dl.dl_boosted = 0; + p->sched_class = &dl_sched_class; + } else if (rt_prio(prio)) { + if (dl_prio(oldprio)) + p->dl.dl_boosted = 0; + if (oldprio < prio) + enqueue_flag = ENQUEUE_HEAD; p->sched_class = &rt_sched_class; - else + } else { + if (dl_prio(oldprio)) + p->dl.dl_boosted = 0; + if (rt_prio(oldprio)) + p->rt.timeout = 0; p->sched_class = &fair_sched_class; + } p->prio = prio; if (running) p->sched_class->set_curr_task(rq); - if (on_rq) - enqueue_task(rq, p, oldprio < prio ? ENQUEUE_HEAD : 0); + if (queued) + enqueue_task(rq, p, enqueue_flag); check_class_changed(rq, p, prev_class, oldprio); out_unlock: @@ -5565,11 +3354,11 @@ void rt_mutex_setprio(struct task_struct *p, int prio) void set_user_nice(struct task_struct *p, long nice) { - int old_prio, delta, on_rq; + int old_prio, delta, queued; unsigned long flags; struct rq *rq; - if (TASK_NICE(p) == nice || nice < -20 || nice > 19) + if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE) return; /* * We have to be careful, if called from sys_setpriority(), @@ -5580,14 +3369,14 @@ void set_user_nice(struct task_struct *p, long nice) * The RT priorities are set via sched_setscheduler(), but we still * allow the 'normal' nice value to be set - but as expected * it wont have any effect on scheduling until the task is - * SCHED_FIFO/SCHED_RR: + * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR: */ - if (task_has_rt_policy(p)) { + if (task_has_dl_policy(p) || task_has_rt_policy(p)) { p->static_prio = NICE_TO_PRIO(nice); goto out_unlock; } - on_rq = p->on_rq; - if (on_rq) + queued = task_on_rq_queued(p); + if (queued) dequeue_task(rq, p, 0); p->static_prio = NICE_TO_PRIO(nice); @@ -5596,14 +3385,14 @@ void set_user_nice(struct task_struct *p, long nice) p->prio = effective_prio(p); delta = p->prio - old_prio; - if (on_rq) { + if (queued) { enqueue_task(rq, p, 0); /* * If the task increased its priority or is running and * lowered its priority, then reschedule its CPU: */ if (delta < 0 || (delta > 0 && task_running(rq, p))) - resched_task(rq->curr); + resched_curr(rq); } out_unlock: task_rq_unlock(rq, p, &flags); @@ -5618,7 +3407,7 @@ EXPORT_SYMBOL(set_user_nice); int can_nice(const struct task_struct *p, const int nice) { /* convert nice value [19,-20] to rlimit style value [1,40] */ - int nice_rlim = 20 - nice; + int nice_rlim = nice_to_rlimit(nice); return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) || capable(CAP_SYS_NICE)); @@ -5642,17 +3431,10 @@ SYSCALL_DEFINE1(nice, int, increment) * We don't have to worry. Conceptually one call occurs first * and we have a single winner. */ - if (increment < -40) - increment = -40; - if (increment > 40) - increment = 40; - - nice = TASK_NICE(current) + increment; - if (nice < -20) - nice = -20; - if (nice > 19) - nice = 19; + increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH); + nice = task_nice(current) + increment; + nice = clamp_val(nice, MIN_NICE, MAX_NICE); if (increment < 0 && !can_nice(current, nice)) return -EPERM; @@ -5670,7 +3452,7 @@ SYSCALL_DEFINE1(nice, int, increment) * task_prio - return the priority value of a given task. * @p: the task in question. * - * This is the priority value as seen by users in /proc. + * Return: The priority value as seen by users in /proc. * RT tasks are offset by -200. Normal tasks are centered * around 0, value goes from -16 to +15. */ @@ -5679,19 +3461,11 @@ int task_prio(const struct task_struct *p) return p->prio - MAX_RT_PRIO; } -/** - * task_nice - return the nice value of a given task. - * @p: the task in question. - */ -int task_nice(const struct task_struct *p) -{ - return TASK_NICE(p); -} -EXPORT_SYMBOL(task_nice); - /** * idle_cpu - is a given cpu idle currently? * @cpu: the processor in question. + * + * Return: 1 if the CPU is currently idle. 0 otherwise. */ int idle_cpu(int cpu) { @@ -5714,6 +3488,8 @@ int idle_cpu(int cpu) /** * idle_task - return the idle task for a given cpu. * @cpu: the processor in question. + * + * Return: The idle task for the cpu @cpu. */ struct task_struct *idle_task(int cpu) { @@ -5723,24 +3499,73 @@ struct task_struct *idle_task(int cpu) /** * find_process_by_pid - find a process with a matching PID value. * @pid: the pid in question. + * + * The task of @pid, if found. %NULL otherwise. */ static struct task_struct *find_process_by_pid(pid_t pid) { return pid ? find_task_by_vpid(pid) : current; } -/* Actually do priority change: must hold pi & rq lock. */ -static void __setscheduler(struct rq *rq, struct task_struct *p, - const struct sched_attr *attr) +/* + * This function initializes the sched_dl_entity of a newly becoming + * SCHED_DEADLINE task. + * + * Only the static values are considered here, the actual runtime and the + * absolute deadline will be properly calculated when the task is enqueued + * for the first time with its new policy. + */ +static void +__setparam_dl(struct task_struct *p, const struct sched_attr *attr) +{ + struct sched_dl_entity *dl_se = &p->dl; + + dl_se->dl_runtime = attr->sched_runtime; + dl_se->dl_deadline = attr->sched_deadline; + dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline; + dl_se->flags = attr->sched_flags; + dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); + + /* + * Changing the parameters of a task is 'tricky' and we're not doing + * the correct thing -- also see task_dead_dl() and switched_from_dl(). + * + * What we SHOULD do is delay the bandwidth release until the 0-lag + * point. This would include retaining the task_struct until that time + * and change dl_overflow() to not immediately decrement the current + * amount. + * + * Instead we retain the current runtime/deadline and let the new + * parameters take effect after the current reservation period lapses. + * This is safe (albeit pessimistic) because the 0-lag point is always + * before the current scheduling deadline. + * + * We can still have temporary overloads because we do not delay the + * change in bandwidth until that time; so admission control is + * not on the safe side. It does however guarantee tasks will never + * consume more than promised. + */ +} + +/* + * sched_setparam() passes in -1 for its policy, to let the functions + * it calls know not to change it. + */ +#define SETPARAM_POLICY -1 + +static void __setscheduler_params(struct task_struct *p, + const struct sched_attr *attr) { int policy = attr->sched_policy; - if (policy == -1) /* setparam */ + if (policy == SETPARAM_POLICY) policy = p->policy; p->policy = policy; - if (fair_policy(policy)) + if (dl_policy(policy)) + __setparam_dl(p, attr); + else if (fair_policy(policy)) p->static_prio = NICE_TO_PRIO(attr->sched_nice); /* @@ -5749,17 +3574,88 @@ static void __setscheduler(struct rq *rq, struct task_struct *p, * getparam()/getattr() don't report silly values for !rt tasks. */ p->rt_priority = attr->sched_priority; - p->normal_prio = normal_prio(p); - p->prio = rt_mutex_getprio(p); + set_load_weight(p); +} + +/* Actually do priority change: must hold pi & rq lock. */ +static void __setscheduler(struct rq *rq, struct task_struct *p, + const struct sched_attr *attr, bool keep_boost) +{ + __setscheduler_params(p, attr); + + /* + * Keep a potential priority boosting if called from + * sched_setscheduler(). + */ +#ifdef TJK_HMP + if (keep_boost) + p->prio = rt_mutex_get_effective_prio(p, normal_prio(p)); + else +#endif + p->prio = normal_prio(p); - if (rt_prio(p->prio)) + if (dl_prio(p->prio)) + p->sched_class = &dl_sched_class; + else if (rt_prio(p->prio)) p->sched_class = &rt_sched_class; else p->sched_class = &fair_sched_class; +} - set_load_weight(p); +static void +__getparam_dl(struct task_struct *p, struct sched_attr *attr) +{ + struct sched_dl_entity *dl_se = &p->dl; + + attr->sched_priority = p->rt_priority; + attr->sched_runtime = dl_se->dl_runtime; + attr->sched_deadline = dl_se->dl_deadline; + attr->sched_period = dl_se->dl_period; + attr->sched_flags = dl_se->flags; +} + +/* + * This function validates the new parameters of a -deadline task. + * We ask for the deadline not being zero, and greater or equal + * than the runtime, as well as the period of being zero or + * greater than deadline. Furthermore, we have to be sure that + * user parameters are above the internal resolution of 1us (we + * check sched_runtime only since it is always the smaller one) and + * below 2^63 ns (we have to check both sched_deadline and + * sched_period, as the latter can be zero). + */ +static bool +__checkparam_dl(const struct sched_attr *attr) +{ + /* deadline != 0 */ + if (attr->sched_deadline == 0) + return false; + + /* + * Since we truncate DL_SCALE bits, make sure we're at least + * that big. + */ + if (attr->sched_runtime < (1ULL << DL_SCALE)) + return false; + + /* + * Since we use the MSB for wrap-around and sign issues, make + * sure it's not set (mind that period can be equal to zero). + */ + if (attr->sched_deadline & (1ULL << 63) || + attr->sched_period & (1ULL << 63)) + return false; + + /* runtime <= deadline <= period (if period != 0) */ + if ((attr->sched_period != 0 && + attr->sched_period < attr->sched_deadline) || + attr->sched_deadline < attr->sched_runtime) + return false; + + return true; } + /* * check the target process has a UID that matches the current process's */ @@ -5780,7 +3676,10 @@ static int __sched_setscheduler(struct task_struct *p, const struct sched_attr *attr, bool user) { - int retval, oldprio, oldpolicy = -1, on_rq, running; + int retval, oldprio, oldpolicy = -1, queued, running; +#ifdef TJK_HMP + int new_effective_prio; +#endif int policy = attr->sched_policy; unsigned long flags; const struct sched_class *prev_class; @@ -5795,13 +3694,20 @@ static int __sched_setscheduler(struct task_struct *p, reset_on_fork = p->sched_reset_on_fork; policy = oldpolicy = p->policy; } else { - reset_on_fork = !!(policy & SCHED_RESET_ON_FORK); - policy &= ~SCHED_RESET_ON_FORK; + reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK); - if (policy != SCHED_FIFO && policy != SCHED_RR && + if (policy != SCHED_DEADLINE && + policy != SCHED_FIFO && policy != SCHED_RR && policy != SCHED_NORMAL && policy != SCHED_BATCH && - policy != SCHED_IDLE) + policy != SCHED_IDLE) { + pr_warn("%s %d:%s policy %d", __func__, p->pid, p->comm, policy); return -EINVAL; + } + } + + if (attr->sched_flags & ~(SCHED_FLAG_RESET_ON_FORK)) { + pr_warn("%s %d:%s sched_flags=%llu", __func__, p->pid, p->comm, attr->sched_flags); + return -EINVAL; } /* @@ -5809,20 +3715,29 @@ static int __sched_setscheduler(struct task_struct *p, * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL, * SCHED_BATCH and SCHED_IDLE is 0. */ - if (attr->sched_priority < 0 || - (p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) || - (!p->mm && attr->sched_priority > MAX_RT_PRIO-1)) + if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) || + (!p->mm && attr->sched_priority > MAX_RT_PRIO-1)) { + pr_warn("%s %d:%s sched_priority=%d", __func__, p->pid, p->comm, attr->sched_priority); return -EINVAL; - if (rt_policy(policy) != (attr->sched_priority != 0)) + } + if ((dl_policy(policy) && !__checkparam_dl(attr)) || + (rt_policy(policy) != (attr->sched_priority != 0))) { + pr_warn("%s %d:%s dl, rt sched_priority=%d", + __func__, p->pid, p->comm, attr->sched_priority); return -EINVAL; + } /* * Allow unprivileged RT tasks to decrease priority: */ if (user && !capable(CAP_SYS_NICE)) { if (fair_policy(policy)) { - if (!can_nice(p, attr->sched_nice)) + if (attr->sched_nice < task_nice(p) && + !can_nice(p, attr->sched_nice)) { + pr_warn("%s %d:%s sched_nice=%d", + __func__, p->pid, p->comm, attr->sched_nice); return -EPERM; + } } if (rt_policy(policy)) { @@ -5830,13 +3745,30 @@ static int __sched_setscheduler(struct task_struct *p, task_rlimit(p, RLIMIT_RTPRIO); /* can't set/change the rt policy */ - if (policy != p->policy && !rlim_rtprio) + if (policy != p->policy && !rlim_rtprio) { + pr_warn("%s %d:%s policy=%d %d %lu", + __func__, p->pid, p->comm, policy, p->policy, rlim_rtprio); return -EPERM; + } /* can't increase priority */ if (attr->sched_priority > p->rt_priority && - attr->sched_priority > rlim_rtprio) + attr->sched_priority > rlim_rtprio){ + pr_warn("%s %d:%s policy=%d %d %lu", __func__, p->pid, p->comm, + attr->sched_priority, p->rt_priority, rlim_rtprio); return -EPERM; + } + } + + /* + * Can't set/change SCHED_DEADLINE policy at all for now + * (safest behavior); in the future we would like to allow + * unprivileged DL tasks to increase their relative deadline + * or reduce their runtime (both ways reducing utilization) + */ + if (dl_policy(policy)) { + pr_warn("%s %d:%s dl policy=%d", __func__, p->pid, p->comm, policy); + return -EPERM; } /* @@ -5844,17 +3776,24 @@ static int __sched_setscheduler(struct task_struct *p, * SCHED_NORMAL if the RLIMIT_NICE would normally permit it. */ if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) { - if (!can_nice(p, TASK_NICE(p))) + if (!can_nice(p, task_nice(p))) { + pr_warn("%s %d:%s dl policy=%d", __func__, p->pid, p->comm, policy); return -EPERM; + } } /* can't change other user's priorities */ - if (!check_same_owner(p)) + if (!check_same_owner(p)) { + pr_warn("%s %d:%s check_same_owner", __func__, p->pid, p->comm); return -EPERM; + } /* Normal users shall not reset the sched_reset_on_fork flag */ - if (p->sched_reset_on_fork && !reset_on_fork) + if (p->sched_reset_on_fork && !reset_on_fork) { + pr_warn("%s %d:%s reset_on_fork=%d %d", + __func__, p->pid, p->comm, p->sched_reset_on_fork, reset_on_fork); return -EPERM; + } } if (user) { @@ -5881,21 +3820,25 @@ static int __sched_setscheduler(struct task_struct *p, } /* - * If not changing anything there's no need to proceed further: + * If not changing anything there's no need to proceed further, + * but store a possible modification of reset_on_fork. */ if (unlikely(policy == p->policy)) { - if (fair_policy(policy) && attr->sched_nice != TASK_NICE(p)) + if (fair_policy(policy) && attr->sched_nice != task_nice(p)) goto change; if (rt_policy(policy) && attr->sched_priority != p->rt_priority) goto change; + if (dl_policy(policy)) + goto change; + p->sched_reset_on_fork = reset_on_fork; task_rq_unlock(rq, p, &flags); return 0; } change: -#ifdef CONFIG_RT_GROUP_SCHED if (user) { +#ifdef CONFIG_RT_GROUP_SCHED /* * Do not allow realtime tasks into groups that have no runtime * assigned. @@ -5904,10 +3847,28 @@ static int __sched_setscheduler(struct task_struct *p, task_group(p)->rt_bandwidth.rt_runtime == 0 && !task_group_is_autogroup(task_group(p))) { task_rq_unlock(rq, p, &flags); + pr_warn("%s rt_runtime", __func__); return -EPERM; } - } #endif +#ifdef CONFIG_SMP + if (dl_bandwidth_enabled() && dl_policy(policy)) { + cpumask_t *span = rq->rd->span; + + /* + * Don't allow tasks with an affinity mask smaller than + * the entire root_domain to become SCHED_DEADLINE. We + * will also fail if there's no bandwidth available. + */ + if (!cpumask_subset(span, &p->cpus_allowed) || + rq->rd->dl_bw.bw == 0) { + task_rq_unlock(rq, p, &flags); + pr_warn("%s allowed", __func__); + return -EPERM; + } + } +#endif + } /* recheck policy now with rq lock held */ if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { @@ -5915,23 +3876,56 @@ static int __sched_setscheduler(struct task_struct *p, task_rq_unlock(rq, p, &flags); goto recheck; } - on_rq = p->on_rq; + + /* + * If setscheduling to SCHED_DEADLINE (or changing the parameters + * of a SCHED_DEADLINE task) we need to check if enough bandwidth + * is available. + */ + if ((dl_policy(policy) || dl_task(p)) && dl_overflow(p, policy, attr)) { + task_rq_unlock(rq, p, &flags); + pr_warn("%s deadline", __func__); + return -EBUSY; + } + + p->sched_reset_on_fork = reset_on_fork; + oldprio = p->prio; + + /* + * Take priority boosted tasks into account. If the new + * effective priority is unchanged, we just store the new + * normal parameters and do not touch the scheduler class and + * the runqueue. This will be done when the task deboost + * itself. + */ +#ifdef TJK_HMP + new_effective_prio = rt_mutex_get_effective_prio(p, newprio); + if (new_effective_prio == oldprio) { + __setscheduler_params(p, attr); + task_rq_unlock(rq, p, &flags); + return 0; + } +#endif + + queued = task_on_rq_queued(p); running = task_current(rq, p); - if (on_rq) + if (queued) dequeue_task(rq, p, 0); if (running) - p->sched_class->put_prev_task(rq, p); - - p->sched_reset_on_fork = reset_on_fork; + put_prev_task(rq, p); - oldprio = p->prio; prev_class = p->sched_class; - __setscheduler(rq, p, attr); + __setscheduler(rq, p, attr, true); if (running) p->sched_class->set_curr_task(rq); - if (on_rq) - enqueue_task(rq, p, 0); + if (queued) { + /* + * We enqueue to tail when the priority of a task is + * increased (user space view). + */ + enqueue_task(rq, p, oldprio <= p->prio ? ENQUEUE_HEAD : 0); + } check_class_changed(rq, p, prev_class, oldprio); task_rq_unlock(rq, p, &flags); @@ -5941,22 +3935,38 @@ static int __sched_setscheduler(struct task_struct *p, return 0; } +static int _sched_setscheduler(struct task_struct *p, int policy, + const struct sched_param *param, bool check) +{ + struct sched_attr attr = { + .sched_policy = policy, + .sched_priority = param->sched_priority, + .sched_nice = PRIO_TO_NICE(p->static_prio), + }; + + /* Fixup the legacy SCHED_RESET_ON_FORK hack. */ + if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) { + attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; + policy &= ~SCHED_RESET_ON_FORK; + attr.sched_policy = policy; + } + + return __sched_setscheduler(p, &attr, check); +} /** * sched_setscheduler - change the scheduling policy and/or RT priority of a thread. * @p: the task in question. * @policy: new policy. * @param: structure containing the new RT priority. * + * Return: 0 on success. An error code otherwise. + * * NOTE that the task may be already dead. */ int sched_setscheduler(struct task_struct *p, int policy, const struct sched_param *param) { - struct sched_attr attr = { - .sched_policy = policy, - .sched_priority = param->sched_priority - }; - return __sched_setscheduler(p, &attr, true); + return _sched_setscheduler(p, policy, param, true); } EXPORT_SYMBOL_GPL(sched_setscheduler); @@ -5976,17 +3986,14 @@ EXPORT_SYMBOL_GPL(sched_setattr); * current context has permission. For example, this is needed in * stop_machine(): we create temporary high priority worker threads, * but our caller might not have that capability. + * + * Return: 0 on success. An error code otherwise. */ int sched_setscheduler_nocheck(struct task_struct *p, int policy, const struct sched_param *param) { - struct sched_attr attr = { - .sched_policy = policy, - .sched_priority = param->sched_priority - }; - return __sched_setscheduler(p, &attr, false); + return _sched_setscheduler(p, policy, param, false); } -EXPORT_SYMBOL(sched_setscheduler_nocheck); static int do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) @@ -6072,15 +4079,13 @@ static int sched_copy_attr(struct sched_attr __user *uattr, * XXX: do we want to be lenient like existing syscalls; or do we want * to be strict and return an error on out-of-bounds values? */ - attr->sched_nice = clamp(attr->sched_nice, -20, 19); + attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE); -out: - return ret; + return 0; err_size: put_user(sizeof(*attr), &uattr->size); - ret = -E2BIG; - goto out; + return -E2BIG; } /** @@ -6088,6 +4093,8 @@ static int sched_copy_attr(struct sched_attr __user *uattr, * @pid: the pid in question. * @policy: new policy. * @param: structure containing the new RT priority. + * + * Return: 0 on success. An error code otherwise. */ SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param) @@ -6103,16 +4110,19 @@ SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, * sys_sched_setparam - set/change the RT priority of a thread * @pid: the pid in question. * @param: structure containing the new RT priority. + * + * Return: 0 on success. An error code otherwise. */ SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param) { - return do_sched_setscheduler(pid, -1, param); + return do_sched_setscheduler(pid, SETPARAM_POLICY, param); } /** * sys_sched_setattr - same as above, but with extended sched_attr * @pid: the pid in question. - * @attr: structure containing the extended parameters. + * @uattr: structure containing the extended parameters. + * @flags: for future extension. */ SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, unsigned int, flags) @@ -6124,8 +4134,12 @@ SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, if (!uattr || pid < 0 || flags) return -EINVAL; - if (sched_copy_attr(uattr, &attr)) - return -EFAULT; + retval = sched_copy_attr(uattr, &attr); + if (retval) + return retval; + + if ((int)attr.sched_policy < 0) + return -EINVAL; rcu_read_lock(); retval = -ESRCH; @@ -6140,6 +4154,9 @@ SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, /** * sys_sched_getscheduler - get the policy (scheduling class) of a thread * @pid: the pid in question. + * + * Return: On success, the policy of the thread. Otherwise, a negative error + * code. */ SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) { @@ -6166,10 +4183,13 @@ SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) * sys_sched_getparam - get the RT priority of a thread * @pid: the pid in question. * @param: structure containing the RT priority. + * + * Return: On success, 0 and the RT priority is in @param. Otherwise, an error + * code. */ SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) { - struct sched_param lp; + struct sched_param lp = { .sched_priority = 0 }; struct task_struct *p; int retval; @@ -6186,7 +4206,8 @@ SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) if (retval) goto out_unlock; - lp.sched_priority = p->rt_priority; + if (task_has_rt_policy(p)) + lp.sched_priority = p->rt_priority; rcu_read_unlock(); /* @@ -6224,29 +4245,25 @@ static int sched_read_attr(struct sched_attr __user *uattr, for (; addr < end; addr++) { if (*addr) - goto err_size; + return -EFBIG; } attr->size = usize; } - ret = copy_to_user(uattr, attr, usize); + ret = copy_to_user(uattr, attr, attr->size); if (ret) return -EFAULT; -out: - return ret; - -err_size: - ret = -E2BIG; - goto out; + return 0; } /** - * sys_sched_getattr - same as above, but with extended "sched_param" + * sys_sched_getattr - similar to sched_getparam, but with sched_attr * @pid: the pid in question. - * @attr: structure containing the extended parameters. + * @uattr: structure containing the extended parameters. * @size: sizeof(attr) for fwd/bwd comp. + * @flags: for future extension. */ SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, unsigned int, size, unsigned int, flags) @@ -6272,10 +4289,14 @@ SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, goto out_unlock; attr.sched_policy = p->policy; - if (task_has_rt_policy(p)) + if (p->sched_reset_on_fork) + attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; + if (task_has_dl_policy(p)) + __getparam_dl(p, &attr); + else if (task_has_rt_policy(p)) attr.sched_priority = p->rt_priority; else - attr.sched_nice = TASK_NICE(p); + attr.sched_nice = task_nice(p); rcu_read_unlock(); @@ -6322,17 +4343,36 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) rcu_read_lock(); if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { rcu_read_unlock(); - goto out_unlock; + goto out_free_new_mask; } rcu_read_unlock(); } retval = security_task_setscheduler(p); if (retval) - goto out_unlock; + goto out_free_new_mask; + cpuset_cpus_allowed(p, cpus_allowed); cpumask_and(new_mask, in_mask, cpus_allowed); + + /* + * Since bandwidth control happens on root_domain basis, + * if admission test is enabled, we only admit -deadline + * tasks allowed to run on all the CPUs in the task's + * root_domain. + */ +#ifdef CONFIG_SMP + if (task_has_dl_policy(p) && dl_bandwidth_enabled()) { + rcu_read_lock(); + if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) { + retval = -EBUSY; + rcu_read_unlock(); + goto out_free_new_mask; + } + rcu_read_unlock(); + } +#endif again: retval = set_cpus_allowed_ptr(p, new_mask); @@ -6348,7 +4388,7 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) goto again; } } -out_unlock: +out_free_new_mask: free_cpumask_var(new_mask); out_free_cpus_allowed: free_cpumask_var(cpus_allowed); @@ -6373,6 +4413,8 @@ static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, * @pid: pid of the process * @len: length in bytes of the bitmask pointed to by user_mask_ptr * @user_mask_ptr: user-space pointer to the new cpu mask + * + * Return: 0 on success. An error code otherwise. */ SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len, unsigned long __user *, user_mask_ptr) @@ -6422,6 +4464,8 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask) * @pid: pid of the process * @len: length in bytes of the bitmask pointed to by user_mask_ptr * @user_mask_ptr: user-space pointer to hold the current cpu mask + * + * Return: 0 on success. An error code otherwise. */ SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, unsigned long __user *, user_mask_ptr) @@ -6456,6 +4500,8 @@ SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, * * This function yields the current CPU to other tasks. If there are no * other threads running on this CPU then this function will return. + * + * Return: 0. */ SYSCALL_DEFINE0(sched_yield) { @@ -6581,12 +4627,12 @@ EXPORT_SYMBOL(yield); * It's the caller's job to ensure that the target task struct * can't go away on us before we can do any checks. * - * Returns: + * Return: * true (>0) if we indeed boosted the target task. * false (0) if we failed to boost the target. * -ESRCH if there's no task to yield to. */ -bool __sched yield_to(struct task_struct *p, bool preempt) +int __sched yield_to(struct task_struct *p, bool preempt) { struct task_struct *curr = current; struct rq *rq, *p_rq; @@ -6608,7 +4654,7 @@ bool __sched yield_to(struct task_struct *p, bool preempt) } double_rq_lock(rq, p_rq); - while (task_rq(p) != p_rq) { + if (task_rq(p) != p_rq) { double_rq_unlock(rq, p_rq); goto again; } @@ -6630,7 +4676,7 @@ bool __sched yield_to(struct task_struct *p, bool preempt) * fairness. */ if (preempt && rq != p_rq) - resched_task(p_rq->curr); + resched_curr(p_rq); } out_unlock: @@ -6679,14 +4725,14 @@ long __sched io_schedule_timeout(long timeout) delayacct_blkio_end(); return ret; } -EXPORT_SYMBOL(io_schedule_timeout); /** * sys_sched_get_priority_max - return maximum RT priority. * @policy: scheduling class. * - * this syscall returns the maximum rt_priority that can be used - * by a given scheduling class. + * Return: On success, this syscall returns the maximum + * rt_priority that can be used by a given scheduling class. + * On failure, a negative error code is returned. */ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) { @@ -6697,6 +4743,7 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) case SCHED_RR: ret = MAX_USER_RT_PRIO-1; break; + case SCHED_DEADLINE: case SCHED_NORMAL: case SCHED_BATCH: case SCHED_IDLE: @@ -6710,8 +4757,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) * sys_sched_get_priority_min - return minimum RT priority. * @policy: scheduling class. * - * this syscall returns the minimum rt_priority that can be used - * by a given scheduling class. + * Return: On success, this syscall returns the minimum + * rt_priority that can be used by a given scheduling class. + * On failure, a negative error code is returned. */ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) { @@ -6722,6 +4770,7 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) case SCHED_RR: ret = 1; break; + case SCHED_DEADLINE: case SCHED_NORMAL: case SCHED_BATCH: case SCHED_IDLE: @@ -6737,6 +4786,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy) * * this syscall writes the default timeslice value of a given process * into the user-space timespec buffer. A value of '0' means infinity. + * + * Return: On success, 0 and the timeslice is in @interval. Otherwise, + * an error code. */ SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, struct timespec __user *, interval) @@ -6762,7 +4814,9 @@ SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, goto out_unlock; rq = task_rq_lock(p, &flags); - time_slice = p->sched_class->get_rr_interval(rq, p); + time_slice = 0; + if (p->sched_class->get_rr_interval) + time_slice = p->sched_class->get_rr_interval(rq, p); task_rq_unlock(rq, p, &flags); rcu_read_unlock(); @@ -6823,7 +4877,7 @@ void show_state_filter(unsigned long state_filter) " task PC stack pid father\n"); #endif rcu_read_lock(); - do_each_thread(g, p) { + for_each_process_thread(g, p) { /* * reset the NMI-timeout, listing all files on a slow * console might take a lot of time: @@ -6831,13 +4885,12 @@ void show_state_filter(unsigned long state_filter) touch_nmi_watchdog(); if (!state_filter || (p->state & state_filter)) sched_show_task(p); - } while_each_thread(g, p); + } touch_all_softlockup_watchdogs(); -#ifdef CONFIG_SYSRQ_SCHED_DEBUG - if (!state_filter) - sysrq_sched_debug_show(); +#ifdef CONFIG_SCHED_DEBUG + sysrq_sched_debug_show(); #endif rcu_read_unlock(); /* @@ -6847,7 +4900,7 @@ void show_state_filter(unsigned long state_filter) debug_show_all_locks(); } -void __cpuinit init_idle_bootup_task(struct task_struct *idle) +void init_idle_bootup_task(struct task_struct *idle) { idle->sched_class = &idle_sched_class; } @@ -6860,22 +4913,14 @@ void __cpuinit init_idle_bootup_task(struct task_struct *idle) * NOTE: this function does not set the idle thread's NEED_RESCHED * flag, to make booting more robust. */ -void __cpuinit init_idle(struct task_struct *idle, int cpu) +void init_idle(struct task_struct *idle, int cpu) { struct rq *rq = cpu_rq(cpu); unsigned long flags; - u64 mark_start; raw_spin_lock_irqsave(&rq->lock, flags); - mark_start = orig_mark_start(idle); - - __sched_fork(idle); - /* - * Restore idle thread's original mark_start as we rely on it being - * correct for maintaining per-cpu counters, curr/prev_runnable_sum. - */ - restore_orig_mark_start(idle, mark_start); + __sched_fork(0, idle); idle->state = TASK_RUNNING; idle->se.exec_start = sched_clock(); @@ -6895,6 +4940,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu) rcu_read_unlock(); rq->curr = rq->idle = idle; + idle->on_rq = TASK_ON_RQ_QUEUED; #if defined(CONFIG_SMP) idle->on_cpu = 1; #endif @@ -6915,162 +4961,56 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu) } #ifdef CONFIG_SMP -void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) -{ - if (p->sched_class && p->sched_class->set_cpus_allowed) - p->sched_class->set_cpus_allowed(p, new_mask); - - cpumask_copy(&p->cpus_allowed, new_mask); - p->nr_cpus_allowed = cpumask_weight(new_mask); -} - -/* - * This is how migration works: - * - * 1) we invoke migration_cpu_stop() on the target CPU using - * stop_one_cpu(). - * 2) stopper starts to run (implicitly forcing the migrated thread - * off the CPU) - * 3) it checks whether the migrated task is still in the wrong runqueue. - * 4) if it's in the wrong runqueue then the migration thread removes - * it and puts it into the right queue. - * 5) stopper completes and stop_one_cpu() returns and the migration - * is done. - */ -/* - * Change a given task's CPU affinity. Migrate the thread to a - * proper CPU and schedule it away if the CPU it's executing on - * is removed from the allowed bitmask. - * - * NOTE: the caller must have a valid reference to the task, the - * task must not exit() & deallocate itself prematurely. The - * call is not atomic; no spinlocks may be held. - */ -int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) +#ifdef CONFIG_NUMA_BALANCING +/* Migrate current task p to target_cpu */ +int migrate_task_to(struct task_struct *p, int target_cpu) { - unsigned long flags; - struct rq *rq; - unsigned int dest_cpu; - int ret = 0; - - rq = task_rq_lock(p, &flags); - - if (cpumask_equal(&p->cpus_allowed, new_mask)) - goto out; - - if (!cpumask_intersects(new_mask, cpu_active_mask)) { - ret = -EINVAL; - goto out; - } + struct migration_arg arg = { p, target_cpu }; + int curr_cpu = task_cpu(p); - do_set_cpus_allowed(p, new_mask); + if (curr_cpu == target_cpu) + return 0; - /* Can the task run on the task's current CPU? If so, we're done */ - if (cpumask_test_cpu(task_cpu(p), new_mask)) - goto out; + if (!cpumask_test_cpu(target_cpu, tsk_cpus_allowed(p))) + return -EINVAL; - dest_cpu = cpumask_any_and(cpu_active_mask, new_mask); - if (p->on_rq) { - struct migration_arg arg = { p, dest_cpu }; - /* Need help from migration thread: drop lock and wait. */ - task_rq_unlock(rq, p, &flags); - stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); - tlb_migrate_finish(p->mm); - return 0; - } -out: - task_rq_unlock(rq, p, &flags); + /* TODO: This is not properly updating schedstats */ - return ret; + trace_sched_move_numa(p, curr_cpu, target_cpu); + return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg); } -EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); /* - * Move (not current) task off this cpu, onto dest cpu. We're doing - * this because either it can't run here any more (set_cpus_allowed() - * away from this CPU, or CPU going down), or because we're - * attempting to rebalance this task on exec (sched_exec). - * - * So we race with normal scheduler movements, but that's OK, as long - * as the task is no longer on this CPU. - * - * Returns non-zero if task was successfully migrated. + * Requeue a task on a given node and accurately track the number of NUMA + * tasks on the runqueues */ -static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) +void sched_setnuma(struct task_struct *p, int nid) { - struct rq *rq_dest, *rq_src; - bool moved = false; - int ret = 0; - - if (unlikely(!cpu_active(dest_cpu))) - return ret; - - rq_src = cpu_rq(src_cpu); - rq_dest = cpu_rq(dest_cpu); - - raw_spin_lock(&p->pi_lock); - double_rq_lock(rq_src, rq_dest); - /* Already moved. */ - if (task_cpu(p) != src_cpu) - goto done; - /* Affinity changed (again). */ - if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) - goto fail; + struct rq *rq; + unsigned long flags; + bool queued, running; - /* - * If we're not on a rq, the next wake-up will ensure we're - * placed properly. - */ - if (p->on_rq) { - dequeue_task(rq_src, p, 0); - set_task_cpu(p, dest_cpu); - enqueue_task(rq_dest, p, 0); - check_preempt_curr(rq_dest, p, 0); - moved = true; - } -done: - ret = 1; -fail: - double_rq_unlock(rq_src, rq_dest); - raw_spin_unlock(&p->pi_lock); - if (moved && !same_freq_domain(src_cpu, dest_cpu)) { - check_for_freq_change(rq_src); - check_for_freq_change(rq_dest); - } - if (moved && task_notify_on_migrate(p)) { - struct migration_notify_data mnd; + rq = task_rq_lock(p, &flags); + queued = task_on_rq_queued(p); + running = task_current(rq, p); - mnd.src_cpu = src_cpu; - mnd.dest_cpu = dest_cpu; - mnd.load = pct_task_load(p); - atomic_notifier_call_chain(&migration_notifier_head, - 0, (void *)&mnd); - } - return ret; -} + if (queued) + dequeue_task(rq, p, 0); + if (running) + put_prev_task(rq, p); -/* - * migration_cpu_stop - this will be executed by a highprio stopper thread - * and performs thread migration by bumping thread off CPU then - * 'pushing' onto another runqueue. - */ -static int migration_cpu_stop(void *data) -{ - struct migration_arg *arg = data; + p->numa_preferred_nid = nid; - /* - * The original target cpu might have gone down and we might - * be on another cpu but it doesn't matter. - */ - local_irq_disable(); - __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu); - local_irq_enable(); - return 0; + if (running) + p->sched_class->set_curr_task(rq); + if (queued) + enqueue_task(rq, p, 0); + task_rq_unlock(rq, p, &flags); } +#endif /* CONFIG_NUMA_BALANCING */ #ifdef CONFIG_HOTPLUG_CPU - /* * Ensures that the idle task is using init_mm right before its cpu goes * offline. @@ -7081,8 +5021,10 @@ void idle_task_exit(void) BUG_ON(cpu_online(smp_processor_id())); - if (mm != &init_mm) + if (mm != &init_mm) { switch_mm(mm, &init_mm, current); + finish_arch_post_lock_switch(); + } mmdrop(mm); } @@ -7100,6 +5042,22 @@ static void calc_load_migrate(struct rq *rq) atomic_long_add(delta, &calc_load_tasks); } +static void put_prev_task_fake(struct rq *rq, struct task_struct *prev) +{ +} + +static const struct sched_class fake_sched_class = { + .put_prev_task = put_prev_task_fake, +}; + +static struct task_struct fake_task = { + /* + * Avoid pull_{rt,dl}_task() + */ + .prio = MAX_PRIO + 1, + .sched_class = &fake_sched_class, +}; + /* * Migrate all tasks from the rq, sleeping tasks will be migrated by * try_to_wake_up()->select_task_rq(). @@ -7125,6 +5083,13 @@ static void migrate_tasks(unsigned int dead_cpu) */ rq->stop = NULL; + /* + * put_prev_task() and pick_next_task() sched + * class method both need to have an up-to-date + * value of rq->clock[_task] + */ + update_rq_clock(rq); + for ( ; ; ) { /* * There's this thread running, bail when that's the only @@ -7133,7 +5098,7 @@ static void migrate_tasks(unsigned int dead_cpu) if (rq->nr_running == 1) break; - next = pick_next_task(rq); + next = pick_next_task(rq, &fake_task); BUG_ON(!next); next->sched_class->put_prev_task(rq, next); @@ -7148,7 +5113,6 @@ static void migrate_tasks(unsigned int dead_cpu) rq->stop = stop; } - #endif /* CONFIG_HOTPLUG_CPU */ #if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL) @@ -7220,10 +5184,61 @@ set_table_entry(struct ctl_table *entry, } } +static struct ctl_table * +sd_alloc_ctl_energy_table(struct sched_group_energy *sge) +{ + struct ctl_table *table = sd_alloc_ctl_entry(5); + + if (table == NULL) + return NULL; + + set_table_entry(&table[0], "nr_idle_states", &sge->nr_idle_states, + sizeof(int), 0644, proc_dointvec_minmax, false); + set_table_entry(&table[1], "idle_states", &sge->idle_states[0].power, + sge->nr_idle_states*sizeof(struct idle_state), 0644, + proc_doulongvec_minmax, false); + set_table_entry(&table[2], "nr_cap_states", &sge->nr_cap_states, + sizeof(int), 0644, proc_dointvec_minmax, false); + set_table_entry(&table[3], "cap_states", &sge->cap_states[0].cap, + sge->nr_cap_states*sizeof(struct capacity_state), 0644, + proc_doulongvec_minmax, false); + + return table; +} + +static struct ctl_table * +sd_alloc_ctl_group_table(struct sched_group *sg) +{ + struct ctl_table *table = sd_alloc_ctl_entry(2); + + if (table == NULL) + return NULL; + + table->procname = kstrdup("energy", GFP_KERNEL); + table->mode = 0555; + table->child = sd_alloc_ctl_energy_table((struct sched_group_energy *)sg->sge); + + return table; +} + static struct ctl_table * sd_alloc_ctl_domain_table(struct sched_domain *sd) { - struct ctl_table *table = sd_alloc_ctl_entry(13); + struct ctl_table *table; + unsigned int nr_entries = 14; + + int i = 0; + struct sched_group *sg = sd->groups; + + if (sg->sge) { + int nr_sgs = 0; + + do {} while (nr_sgs++, sg = sg->next, sg != sd->groups); + + nr_entries += nr_sgs; + } + + table = sd_alloc_ctl_entry(nr_entries); if (table == NULL) return NULL; @@ -7251,14 +5266,29 @@ sd_alloc_ctl_domain_table(struct sched_domain *sd) sizeof(int), 0644, proc_dointvec_minmax, false); set_table_entry(&table[10], "flags", &sd->flags, sizeof(int), 0644, proc_dointvec_minmax, false); - set_table_entry(&table[11], "name", sd->name, + set_table_entry(&table[11], "max_newidle_lb_cost", + &sd->max_newidle_lb_cost, + sizeof(long), 0644, proc_doulongvec_minmax, false); + set_table_entry(&table[12], "name", sd->name, CORENAME_MAX_SIZE, 0444, proc_dostring, false); - /* &table[12] is terminator */ + sg = sd->groups; + if (sg->sge) { + char buf[32]; + struct ctl_table *entry = &table[13]; + + do { + snprintf(buf, 32, "group%d", i); + entry->procname = kstrdup(buf, GFP_KERNEL); + entry->mode = 0555; + entry->child = sd_alloc_ctl_group_table(sg); + } while (entry++, i++, sg = sg->next, sg != sd->groups); + } + /* &table[nr_entries-1] is terminator */ return table; } -static ctl_table *sd_alloc_ctl_cpu_table(int cpu) +static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu) { struct ctl_table *entry, *table; struct sched_domain *sd; @@ -7324,7 +5354,7 @@ static void register_sched_domain_sysctl(void) static void unregister_sched_domain_sysctl(void) { } -#endif +#endif /* CONFIG_SCHED_DEBUG && CONFIG_SYSCTL */ static void set_rq_online(struct rq *rq) { @@ -7360,7 +5390,7 @@ static void set_rq_offline(struct rq *rq) * migration_call - callback that gets triggered when a CPU is added. * Here we can start up the necessary migration thread for the new CPU. */ -static int __cpuinit +static int migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) { int cpu = (long)hcpu; @@ -7370,11 +5400,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) switch (action & ~CPU_TASKS_FROZEN) { case CPU_UP_PREPARE: - raw_spin_lock_irqsave(&rq->lock, flags); - set_window_start(rq); - raw_spin_unlock_irqrestore(&rq->lock, flags); rq->calc_load_update = calc_load_update; - account_reset_rq(rq); break; case CPU_ONLINE: @@ -7393,8 +5419,6 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) sched_ttwu_pending(); /* Update our root-domain */ raw_spin_lock_irqsave(&rq->lock, flags); - migrate_sync_cpu(cpu); - if (rq->rd) { BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); set_rq_offline(rq); @@ -7405,7 +5429,6 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) break; case CPU_DEAD: - clear_hmp_request(cpu); calc_load_migrate(rq); break; #endif @@ -7421,15 +5444,25 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) * happens before everything else. This has to be lower priority than * the notifier in the perf_event subsystem, though. */ -static struct notifier_block __cpuinitdata migration_notifier = { +static struct notifier_block migration_notifier = { .notifier_call = migration_call, .priority = CPU_PRI_MIGRATION, }; -static int __cpuinit sched_cpu_active(struct notifier_block *nfb, +static void set_cpu_rq_start_time(void) +{ + int cpu = smp_processor_id(); + struct rq *rq = cpu_rq(cpu); + rq->age_stamp = sched_clock_cpu(cpu); +} + +static int sched_cpu_active(struct notifier_block *nfb, unsigned long action, void *hcpu) { switch (action & ~CPU_TASKS_FROZEN) { + case CPU_STARTING: + set_cpu_rq_start_time(); + return NOTIFY_OK; case CPU_DOWN_FAILED: set_cpu_active((long)hcpu, true); return NOTIFY_OK; @@ -7438,16 +5471,39 @@ static int __cpuinit sched_cpu_active(struct notifier_block *nfb, } } -static int __cpuinit sched_cpu_inactive(struct notifier_block *nfb, +static int sched_cpu_inactive(struct notifier_block *nfb, unsigned long action, void *hcpu) { + unsigned long flags; + long cpu = (long)hcpu; + struct dl_bw *dl_b; + switch (action & ~CPU_TASKS_FROZEN) { case CPU_DOWN_PREPARE: - set_cpu_active((long)hcpu, false); + set_cpu_active(cpu, false); + + /* explicitly allow suspend */ + if (!(action & CPU_TASKS_FROZEN)) { + bool overflow; + int cpus; + + rcu_read_lock_sched(); + dl_b = dl_bw_of(cpu); + + raw_spin_lock_irqsave(&dl_b->lock, flags); + cpus = dl_bw_cpus(cpu); + overflow = __dl_overflow(dl_b, cpus, 0, 0); + raw_spin_unlock_irqrestore(&dl_b->lock, flags); + + rcu_read_unlock_sched(); + + if (overflow) + return notifier_from_errno(-EBUSY); + } return NOTIFY_OK; - default: - return NOTIFY_DONE; } + + return NOTIFY_DONE; } static int __init migration_init(void) @@ -7468,9 +5524,6 @@ static int __init migration_init(void) return 0; } early_initcall(migration_init); -#endif - -#ifdef CONFIG_SMP static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */ @@ -7504,9 +5557,6 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, if (!(sd->flags & SD_LOAD_BALANCE)) { printk("does not load-balance\n"); - if (sd->parent) - printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain" - " has parent"); return -1; } @@ -7529,18 +5579,6 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, break; } - /* - * Even though we initialize ->power to something semi-sane, - * we leave power_orig unset. This allows us to detect if - * domain iteration is still funny without causing /0 traps. - */ - if (!group->sgp->power_orig) { - printk(KERN_CONT "\n"); - printk(KERN_ERR "ERROR: domain->cpu_power not " - "set\n"); - break; - } - if (!cpumask_weight(sched_group_cpus(group))) { printk(KERN_CONT "\n"); printk(KERN_ERR "ERROR: empty group\n"); @@ -7559,9 +5597,9 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group)); printk(KERN_CONT " %s", str); - if (group->sgp->power != SCHED_POWER_SCALE) { - printk(KERN_CONT " (cpu_power = %d)", - group->sgp->power); + if (group->sgc->capacity != SCHED_CAPACITY_SCALE) { + printk(KERN_CONT " (cpu_capacity = %ld)", + group->sgc->capacity); } group = group->next; @@ -7611,16 +5649,22 @@ static inline bool sched_debug(void) static int sd_degenerate(struct sched_domain *sd) { - if (cpumask_weight(sched_domain_span(sd)) == 1) - return 1; + if (cpumask_weight(sched_domain_span(sd)) == 1) { + if (sd->groups->sge) + sd->flags &= ~SD_LOAD_BALANCE; + else + return 1; + } /* Following flags need at least 2 groups */ if (sd->flags & (SD_LOAD_BALANCE | SD_BALANCE_NEWIDLE | SD_BALANCE_FORK | SD_BALANCE_EXEC | - SD_SHARE_CPUPOWER | - SD_SHARE_PKG_RESOURCES)) { + SD_SHARE_CPUCAPACITY | + SD_SHARE_PKG_RESOURCES | + SD_SHARE_POWERDOMAIN | + SD_SHARE_CAP_STATES)) { if (sd->groups != sd->groups->next) return 0; } @@ -7649,8 +5693,16 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) SD_BALANCE_NEWIDLE | SD_BALANCE_FORK | SD_BALANCE_EXEC | - SD_SHARE_CPUPOWER | - SD_SHARE_PKG_RESOURCES); + SD_SHARE_CPUCAPACITY | + SD_SHARE_PKG_RESOURCES | + SD_PREFER_SIBLING | + SD_SHARE_POWERDOMAIN | + SD_SHARE_CAP_STATES); + if (parent->groups->sge) { + parent->flags &= ~SD_LOAD_BALANCE; + return 0; + } + if (nr_node_ids == 1) pflags &= ~SD_SERIALIZE; } @@ -7665,6 +5717,8 @@ static void free_rootdomain(struct rcu_head *rcu) struct root_domain *rd = container_of(rcu, struct root_domain, rcu); cpupri_cleanup(&rd->cpupri); + cpudl_cleanup(&rd->cpudl); + free_cpumask_var(rd->dlo_mask); free_cpumask_var(rd->rto_mask); free_cpumask_var(rd->online); free_cpumask_var(rd->span); @@ -7687,7 +5741,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd) cpumask_clear_cpu(rq->cpu, old_rd->span); /* - * If we dont want to free the old_rt yet then + * If we dont want to free the old_rd yet then * set old_rd to NULL to skip the freeing later * in this function: */ @@ -7716,8 +5770,14 @@ static int init_rootdomain(struct root_domain *rd) goto out; if (!alloc_cpumask_var(&rd->online, GFP_KERNEL)) goto free_span; - if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL)) + if (!alloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL)) goto free_online; + if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL)) + goto free_dlo_mask; + + init_dl_bw(&rd->dl_bw); + if (cpudl_init(&rd->cpudl) != 0) + goto free_dlo_mask; if (cpupri_init(&rd->cpupri) != 0) goto free_rto_mask; @@ -7725,6 +5785,8 @@ static int init_rootdomain(struct root_domain *rd) free_rto_mask: free_cpumask_var(rd->rto_mask); +free_dlo_mask: + free_cpumask_var(rd->dlo_mask); free_online: free_cpumask_var(rd->online); free_span: @@ -7762,7 +5824,7 @@ static struct root_domain *alloc_rootdomain(void) return rd; } -static void free_sched_groups(struct sched_group *sg, int free_sgp) +static void free_sched_groups(struct sched_group *sg, int free_sgc) { struct sched_group *tmp, *first; @@ -7773,8 +5835,11 @@ static void free_sched_groups(struct sched_group *sg, int free_sgp) do { tmp = sg->next; - if (free_sgp && atomic_dec_and_test(&sg->sgp->ref)) - kfree(sg->sgp); + if (free_sgc && atomic_dec_and_test(&sg->sgc->ref)) + kfree(sg->sgc); + + if (free_sgc && atomic_dec_and_test(&sg->sge->ref)) + kfree(sg->sge); kfree(sg); sg = tmp; @@ -7792,7 +5857,8 @@ static void free_sched_domain(struct rcu_head *rcu) if (sd->flags & SD_OVERLAP) { free_sched_groups(sd->groups, 1); } else if (atomic_dec_and_test(&sd->groups->ref)) { - kfree(sd->groups->sgp); + kfree(sd->groups->sgc); + kfree(sd->groups->sge); kfree(sd->groups); } kfree(sd); @@ -7819,19 +5885,45 @@ static void destroy_sched_domains(struct sched_domain *sd, int cpu) * two cpus are in the same cache domain, see cpus_share_cache(). */ DEFINE_PER_CPU(struct sched_domain *, sd_llc); +DEFINE_PER_CPU(int, sd_llc_size); DEFINE_PER_CPU(int, sd_llc_id); +DEFINE_PER_CPU(struct sched_domain *, sd_numa); +DEFINE_PER_CPU(struct sched_domain *, sd_busy); +DEFINE_PER_CPU(struct sched_domain *, sd_asym); +DEFINE_PER_CPU(struct sched_domain *, sd_ea); static void update_top_cache_domain(int cpu) { struct sched_domain *sd; + struct sched_domain *busy_sd = NULL, *ea_sd = NULL; int id = cpu; + int size = 1; sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES); - if (sd) + if (sd) { id = cpumask_first(sched_domain_span(sd)); + size = cpumask_weight(sched_domain_span(sd)); + busy_sd = sd->parent; /* sd_busy */ + } + rcu_assign_pointer(per_cpu(sd_busy, cpu), busy_sd); rcu_assign_pointer(per_cpu(sd_llc, cpu), sd); + per_cpu(sd_llc_size, cpu) = size; per_cpu(sd_llc_id, cpu) = id; + + sd = lowest_flag_domain(cpu, SD_NUMA); + rcu_assign_pointer(per_cpu(sd_numa, cpu), sd); + + sd = highest_flag_domain(cpu, SD_ASYM_PACKING); + rcu_assign_pointer(per_cpu(sd_asym, cpu), sd); + + for_each_domain(cpu, sd) { + if (sd->groups->sge) + ea_sd = sd; + else + break; + } + rcu_assign_pointer(per_cpu(sd_ea, cpu), ea_sd); } /* @@ -7843,7 +5935,6 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) { struct rq *rq = cpu_rq(cpu); struct sched_domain *tmp; - unsigned long next_balance = rq->next_balance; /* Remove the sched domains which do not contribute to scheduling. */ for (tmp = sd; tmp; ) { @@ -7855,6 +5946,13 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) tmp->parent = parent->parent; if (parent->parent) parent->parent->child = tmp; + /* + * Transfer SD_PREFER_SIBLING down in case of a + * degenerate parent; the spans match for this + * so the property transfers. + */ + if (parent->flags & SD_PREFER_SIBLING) + tmp->flags |= SD_PREFER_SIBLING; destroy_sched_domain(parent, cpu); } else tmp = tmp->parent; @@ -7868,17 +5966,6 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) sd->child = NULL; } - for (tmp = sd; tmp; ) { - unsigned long interval; - - interval = msecs_to_jiffies(tmp->balance_interval); - if (time_after(next_balance, tmp->last_balance + interval)) - next_balance = tmp->last_balance + interval; - - tmp = tmp->parent; - } - rq->next_balance = next_balance; - sched_domain_debug(sd, cpu); rq_attach_root(rq, rd); @@ -7902,17 +5989,6 @@ static int __init isolated_cpu_setup(char *str) __setup("isolcpus=", isolated_cpu_setup); -static const struct cpumask *cpu_cpu_mask(int cpu) -{ - return cpumask_of_node(cpu_to_node(cpu)); -} - -struct sd_data { - struct sched_domain **__percpu sd; - struct sched_group **__percpu sg; - struct sched_group_power **__percpu sgp; -}; - struct s_data { struct sched_domain ** __percpu sd; struct root_domain *rd; @@ -7925,21 +6001,6 @@ enum s_alloc { sa_none, }; -struct sched_domain_topology_level; - -typedef struct sched_domain *(*sched_domain_init_f)(struct sched_domain_topology_level *tl, int cpu); -typedef const struct cpumask *(*sched_domain_mask_f)(int cpu); - -#define SDTL_OVERLAP 0x01 - -struct sched_domain_topology_level { - sched_domain_init_f init; - sched_domain_mask_f mask; - int flags; - int numa_level; - struct sd_data data; -}; - /* * Build an iteration mask that can exclude certain CPUs from the upwards * domain traversal. @@ -7985,7 +6046,7 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu) const struct cpumask *span = sched_domain_span(sd); struct cpumask *covered = sched_domains_tmpmask; struct sd_data *sdd = sd->private; - struct sched_domain *child; + struct sched_domain *sibling; int i; cpumask_clear(covered); @@ -7996,10 +6057,10 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu) if (cpumask_test_cpu(i, covered)) continue; - child = *per_cpu_ptr(sdd->sd, i); + sibling = *per_cpu_ptr(sdd->sd, i); /* See the comment near build_group_mask(). */ - if (!cpumask_test_cpu(i, sched_domain_span(child))) + if (!cpumask_test_cpu(i, sched_domain_span(sibling))) continue; sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(), @@ -8009,24 +6070,26 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu) goto fail; sg_span = sched_group_cpus(sg); - if (child->child) { - child = child->child; - cpumask_copy(sg_span, sched_domain_span(child)); - } else + if (sibling->child) + cpumask_copy(sg_span, sched_domain_span(sibling->child)); + else cpumask_set_cpu(i, sg_span); cpumask_or(covered, covered, sg_span); - sg->sgp = *per_cpu_ptr(sdd->sgp, i); - if (atomic_inc_return(&sg->sgp->ref) == 1) + sg->sgc = *per_cpu_ptr(sdd->sgc, i); + if (atomic_inc_return(&sg->sgc->ref) == 1) build_group_mask(sd, sg); /* - * Initialize sgp->power such that even if we mess up the + * Initialize sgc->capacity such that even if we mess up the * domains and no possible iteration will get us here, we won't * die on a /0 trap. */ - sg->sgp->power = SCHED_POWER_SCALE * cpumask_weight(sg_span); + sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span); + sg->sgc->max_capacity = SCHED_CAPACITY_SCALE; + + sg->sge = *per_cpu_ptr(sdd->sge, i); /* * Make sure the first group of this domain contains the @@ -8064,8 +6127,9 @@ static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg) if (sg) { *sg = *per_cpu_ptr(sdd->sg, cpu); - (*sg)->sgp = *per_cpu_ptr(sdd->sgp, cpu); - atomic_set(&(*sg)->sgp->ref, 1); /* for claim_allocations */ + (*sg)->sgc = *per_cpu_ptr(sdd->sgc, cpu); + atomic_set(&(*sg)->sgc->ref, 1); /* for claim_allocations */ + (*sg)->sge = *per_cpu_ptr(sdd->sge, cpu); } return cpu; @@ -8074,7 +6138,7 @@ static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg) /* * build_sched_groups will build a circular linked list of the groups * covered by the given span, and will set each group's ->cpumask correctly, - * and ->cpu_power to 0. + * and ->cpu_capacity to 0. * * Assumes the sched_domain tree is fully constructed */ @@ -8090,7 +6154,7 @@ build_sched_groups(struct sched_domain *sd, int cpu) get_group(cpu, sdd, &sd->groups); atomic_inc(&sd->groups->ref); - if (cpu != cpumask_first(sched_domain_span(sd))) + if (cpu != cpumask_first(span)) return 0; lockdep_assert_held(&sched_domains_mutex); @@ -8100,14 +6164,12 @@ build_sched_groups(struct sched_domain *sd, int cpu) for_each_cpu(i, span) { struct sched_group *sg; - int group = get_group(i, sdd, &sg); - int j; + int group, j; if (cpumask_test_cpu(i, covered)) continue; - cpumask_clear(sched_group_cpus(sg)); - sg->sgp->power = 0; + group = get_group(i, sdd, &sg); cpumask_setall(sched_group_mask(sg)); for_each_cpu(j, span) { @@ -8130,20 +6192,20 @@ build_sched_groups(struct sched_domain *sd, int cpu) } /* - * Initialize sched groups cpu_power. + * Initialize sched groups cpu_capacity. * - * cpu_power indicates the capacity of sched group, which is used while + * cpu_capacity indicates the capacity of sched group, which is used while * distributing the load between different sched groups in a sched domain. - * Typically cpu_power for all the groups in a sched domain will be same unless - * there are asymmetries in the topology. If there are asymmetries, group - * having more cpu_power will pickup more load compared to the group having - * less cpu_power. + * Typically cpu_capacity for all the groups in a sched domain will be same + * unless there are asymmetries in the topology. If there are asymmetries, + * group having more cpu_capacity will pickup more load compared to the + * group having less cpu_capacity. */ -static void init_sched_groups_power(int cpu, struct sched_domain *sd) +static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) { struct sched_group *sg = sd->groups; - WARN_ON(!sd || !sg); + WARN_ON(!sg); do { sg->group_weight = cpumask_weight(sched_group_cpus(sg)); @@ -8153,47 +6215,72 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd) if (cpu != group_balance_cpu(sg)) return; - update_group_power(sd, cpu); - atomic_set(&sg->sgp->nr_busy_cpus, sg->group_weight); + update_group_capacity(sd, cpu); + atomic_set(&sg->sgc->nr_busy_cpus, sg->group_weight); } -int __weak arch_sd_sibling_asym_packing(void) +static void init_sched_energy(int cpu, struct sched_domain *sd, + struct sched_domain_topology_level *tl) { - return 0*SD_ASYM_PACKING; -} - -/* - * Initializers for schedule domains - * Non-inlined to reduce accumulated stack pressure in build_sched_domains() - */ + struct sched_group *sg = sd->groups; + struct sched_group_energy *sge = sg->sge; + sched_domain_energy_f fn = tl->energy; + struct cpumask *mask = sched_group_cpus(sg); + int nr_idle_states_below = 0; + if (fn && sd->child && !sd->child->groups->sge) { + pr_err("BUG: EAS setup broken for CPU%d\n", cpu); #ifdef CONFIG_SCHED_DEBUG -# define SD_INIT_NAME(sd, type) sd->name = #type -#else -# define SD_INIT_NAME(sd, type) do { } while (0) + pr_err(" energy data on %s but not on %s domain\n", + sd->name, sd->child->name); #endif + return; + } -#define SD_INIT_FUNC(type) \ -static noinline struct sched_domain * \ -sd_init_##type(struct sched_domain_topology_level *tl, int cpu) \ -{ \ - struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu); \ - *sd = SD_##type##_INIT; \ - SD_INIT_NAME(sd, type); \ - sd->private = &tl->data; \ - return sd; \ + if (cpu != group_balance_cpu(sg)) + return; + + if (!fn || !fn(cpu)) { + sg->sge = NULL; + return; + } + + atomic_set(&sg->sge->ref, 1); /* for claim_allocations */ + + if (cpumask_weight(mask) > 1) + check_sched_energy_data(cpu, fn, mask); + + /* Figure out the number of true cpuidle states below current group */ + sd = sd->child; + for_each_lower_domain(sd) { + nr_idle_states_below += sd->groups->sge->nr_idle_states; + + /* Disregard non-cpuidle 'active' idle states */ + if (sd->child) + nr_idle_states_below--; + } + + sge->nr_idle_states = fn(cpu)->nr_idle_states; + sge->nr_idle_states_below = nr_idle_states_below; + sge->nr_cap_states = fn(cpu)->nr_cap_states; + sge->idle_states = (struct idle_state *) + ((void *)&sge->cap_states + + sizeof(sge->cap_states)); + sge->cap_states = (struct capacity_state *) + ((void *)&sge->cap_states + + sizeof(sge->cap_states) + + sge->nr_idle_states * + sizeof(struct idle_state)); + memcpy(sge->idle_states, fn(cpu)->idle_states, + sge->nr_idle_states*sizeof(struct idle_state)); + memcpy(sge->cap_states, fn(cpu)->cap_states, + sge->nr_cap_states*sizeof(struct capacity_state)); } -SD_INIT_FUNC(CPU) -#ifdef CONFIG_SCHED_SMT - SD_INIT_FUNC(SIBLING) -#endif -#ifdef CONFIG_SCHED_MC - SD_INIT_FUNC(MC) -#endif -#ifdef CONFIG_SCHED_BOOK - SD_INIT_FUNC(BOOK) -#endif +/* + * Initializers for schedule domains + * Non-inlined to reduce accumulated stack pressure in build_sched_domains() + */ static int default_relax_domain_level = -1; int sched_domain_level_max; @@ -8278,97 +6365,165 @@ static void claim_allocations(int cpu, struct sched_domain *sd) if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref)) *per_cpu_ptr(sdd->sg, cpu) = NULL; - if (atomic_read(&(*per_cpu_ptr(sdd->sgp, cpu))->ref)) - *per_cpu_ptr(sdd->sgp, cpu) = NULL; -} + if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref)) + *per_cpu_ptr(sdd->sgc, cpu) = NULL; -#ifdef CONFIG_SCHED_SMT -static const struct cpumask *cpu_smt_mask(int cpu) -{ - return topology_thread_cpumask(cpu); + if (atomic_read(&(*per_cpu_ptr(sdd->sge, cpu))->ref)) + *per_cpu_ptr(sdd->sge, cpu) = NULL; } -#endif - -/* - * Topology list, bottom-up. - */ -static struct sched_domain_topology_level default_topology[] = { -#ifdef CONFIG_SCHED_SMT - { sd_init_SIBLING, cpu_smt_mask, }, -#endif -#ifdef CONFIG_SCHED_MC - { sd_init_MC, cpu_coregroup_mask, }, -#endif -#ifdef CONFIG_SCHED_BOOK - { sd_init_BOOK, cpu_book_mask, }, -#endif - { sd_init_CPU, cpu_cpu_mask, }, - { NULL, }, -}; - -static struct sched_domain_topology_level *sched_domain_topology = default_topology; #ifdef CONFIG_NUMA - static int sched_domains_numa_levels; static int *sched_domains_numa_distance; static struct cpumask ***sched_domains_numa_masks; static int sched_domains_curr_level; +#endif -static inline int sd_local_flags(int level) -{ - if (sched_domains_numa_distance[level] > RECLAIM_DISTANCE) - return 0; - - return SD_BALANCE_EXEC | SD_BALANCE_FORK | SD_WAKE_AFFINE; -} +/* + * SD_flags allowed in topology descriptions. + * + * SD_SHARE_CPUCAPACITY - describes SMT topologies + * SD_SHARE_PKG_RESOURCES - describes shared caches + * SD_NUMA - describes NUMA topologies + * SD_SHARE_POWERDOMAIN - describes shared power domain + * SD_SHARE_CAP_STATES - describes shared capacity states + * + * Odd one out: + * SD_ASYM_PACKING - describes SMT quirks + */ +#define TOPOLOGY_SD_FLAGS \ + (SD_SHARE_CPUCAPACITY | \ + SD_SHARE_PKG_RESOURCES | \ + SD_NUMA | \ + SD_ASYM_PACKING | \ + SD_SHARE_POWERDOMAIN | \ + SD_SHARE_CAP_STATES) static struct sched_domain * -sd_numa_init(struct sched_domain_topology_level *tl, int cpu) +sd_init(struct sched_domain_topology_level *tl, int cpu) { struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu); - int level = tl->numa_level; - int sd_weight = cpumask_weight( - sched_domains_numa_masks[level][cpu_to_node(cpu)]); + int sd_weight, sd_flags = 0; + +#ifdef CONFIG_NUMA + /* + * Ugly hack to pass state to sd_numa_mask()... + */ + sched_domains_curr_level = tl->numa_level; +#endif + + sd_weight = cpumask_weight(tl->mask(cpu)); + + if (tl->sd_flags) + sd_flags = (*tl->sd_flags)(); + if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS, + "wrong sd_flags in topology description\n")) + sd_flags &= ~TOPOLOGY_SD_FLAGS; *sd = (struct sched_domain){ .min_interval = sd_weight, .max_interval = 2*sd_weight, .busy_factor = 32, .imbalance_pct = 125, - .cache_nice_tries = 2, - .busy_idx = 3, - .idle_idx = 2, + + .cache_nice_tries = 0, + .busy_idx = 0, + .idle_idx = 0, .newidle_idx = 0, .wake_idx = 0, .forkexec_idx = 0, .flags = 1*SD_LOAD_BALANCE | 1*SD_BALANCE_NEWIDLE - | 0*SD_BALANCE_EXEC - | 0*SD_BALANCE_FORK - | 0*SD_BALANCE_WAKE - | 0*SD_WAKE_AFFINE - | 0*SD_SHARE_CPUPOWER + | 1*SD_BALANCE_EXEC + | 1*SD_BALANCE_FORK + | 1*SD_BALANCE_WAKE + | 1*SD_WAKE_AFFINE + | 0*SD_SHARE_CPUCAPACITY | 0*SD_SHARE_PKG_RESOURCES - | 1*SD_SERIALIZE + | 0*SD_SERIALIZE | 0*SD_PREFER_SIBLING - | sd_local_flags(level) + | 0*SD_NUMA + | sd_flags , + .last_balance = jiffies, .balance_interval = sd_weight, + .smt_gain = 0, + .max_newidle_lb_cost = 0, + .next_decay_max_lb_cost = jiffies, +#ifdef CONFIG_SCHED_DEBUG + .name = tl->name, +#endif }; - SD_INIT_NAME(sd, NUMA); - sd->private = &tl->data; /* - * Ugly hack to pass state to sd_numa_mask()... + * Convert topological properties into behaviour. */ - sched_domains_curr_level = tl->numa_level; + + if (sd->flags & SD_SHARE_CPUCAPACITY) { + sd->imbalance_pct = 110; + sd->smt_gain = 1178; /* ~15% */ + + } else if (sd->flags & SD_SHARE_PKG_RESOURCES) { + sd->imbalance_pct = 117; + sd->cache_nice_tries = 1; + sd->busy_idx = 2; + +#ifdef CONFIG_NUMA + } else if (sd->flags & SD_NUMA) { + sd->cache_nice_tries = 2; + sd->busy_idx = 3; + sd->idle_idx = 2; + + sd->flags |= SD_SERIALIZE; + if (sched_domains_numa_distance[tl->numa_level] > RECLAIM_DISTANCE) { + sd->flags &= ~(SD_BALANCE_EXEC | + SD_BALANCE_FORK | + SD_WAKE_AFFINE); + } + +#endif + } else { + sd->flags |= SD_PREFER_SIBLING; + sd->cache_nice_tries = 1; + sd->busy_idx = 2; + sd->idle_idx = 1; + } + + sd->private = &tl->data; return sd; } +/* + * Topology list, bottom-up. + */ +static struct sched_domain_topology_level default_topology[] = { +#ifdef CONFIG_SCHED_SMT + { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) }, +#endif +#ifdef CONFIG_SCHED_MC + { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) }, +#endif +#ifdef TJK_HMP + { sd_init_CPU, cpu_cpu_mask, }, +#endif + { NULL, }, +}; + +struct sched_domain_topology_level *sched_domain_topology = default_topology; + +#define for_each_sd_topology(tl) \ + for (tl = sched_domain_topology; tl->mask; tl++) + +void set_sched_topology(struct sched_domain_topology_level *tl) +{ + sched_domain_topology = tl; +} + +#ifdef CONFIG_NUMA + static const struct cpumask *sd_numa_mask(int cpu) { return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)]; @@ -8463,6 +6618,10 @@ static void sched_init_numa(void) if (!sched_debug()) break; } + + if (!level) + return; + /* * 'level' contains the number of unique distances, excluding the * identity distance node_distance(i,i). @@ -8512,7 +6671,10 @@ static void sched_init_numa(void) } } - tl = kzalloc((ARRAY_SIZE(default_topology) + level) * + /* Compute default topology size */ + for (i = 0; sched_domain_topology[i].mask; i++); + + tl = kzalloc((i + level + 1) * sizeof(struct sched_domain_topology_level), GFP_KERNEL); if (!tl) return; @@ -8520,18 +6682,19 @@ static void sched_init_numa(void) /* * Copy the default topology bits.. */ - for (i = 0; default_topology[i].init; i++) - tl[i] = default_topology[i]; + for (i = 0; sched_domain_topology[i].mask; i++) + tl[i] = sched_domain_topology[i]; /* * .. and append 'j' levels of NUMA goodness. */ for (j = 0; j < level; i++, j++) { tl[i] = (struct sched_domain_topology_level){ - .init = sd_numa_init, .mask = sd_numa_mask, + .sd_flags = cpu_numa_flags, .flags = SDTL_OVERLAP, .numa_level = j, + SD_INIT_NAME(NUMA) }; } @@ -8605,7 +6768,7 @@ static int __sdt_alloc(const struct cpumask *cpu_map) struct sched_domain_topology_level *tl; int j; - for (tl = sched_domain_topology; tl->init; tl++) { + for_each_sd_topology(tl) { struct sd_data *sdd = &tl->data; sdd->sd = alloc_percpu(struct sched_domain *); @@ -8616,16 +6779,30 @@ static int __sdt_alloc(const struct cpumask *cpu_map) if (!sdd->sg) return -ENOMEM; - sdd->sgp = alloc_percpu(struct sched_group_power *); - if (!sdd->sgp) + sdd->sgc = alloc_percpu(struct sched_group_capacity *); + if (!sdd->sgc) + return -ENOMEM; + + sdd->sge = alloc_percpu(struct sched_group_energy *); + if (!sdd->sge) return -ENOMEM; for_each_cpu(j, cpu_map) { struct sched_domain *sd; struct sched_group *sg; - struct sched_group_power *sgp; + struct sched_group_capacity *sgc; + struct sched_group_energy *sge; + sched_domain_energy_f fn = tl->energy; + unsigned int nr_idle_states = 0; + unsigned int nr_cap_states = 0; + + if (fn && fn(j)) { + nr_idle_states = fn(j)->nr_idle_states; + nr_cap_states = fn(j)->nr_cap_states; + BUG_ON(!nr_idle_states || !nr_cap_states); + } - sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(), + sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(), GFP_KERNEL, cpu_to_node(j)); if (!sd) return -ENOMEM; @@ -8641,12 +6818,22 @@ static int __sdt_alloc(const struct cpumask *cpu_map) *per_cpu_ptr(sdd->sg, j) = sg; - sgp = kzalloc_node(sizeof(struct sched_group_power) + cpumask_size(), + sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(), GFP_KERNEL, cpu_to_node(j)); - if (!sgp) + if (!sgc) + return -ENOMEM; + + *per_cpu_ptr(sdd->sgc, j) = sgc; + + sge = kzalloc_node(sizeof(struct sched_group_energy) + + nr_idle_states*sizeof(struct idle_state) + + nr_cap_states*sizeof(struct capacity_state), + GFP_KERNEL, cpu_to_node(j)); + + if (!sge) return -ENOMEM; - *per_cpu_ptr(sdd->sgp, j) = sgp; + *per_cpu_ptr(sdd->sge, j) = sge; } } @@ -8658,7 +6845,7 @@ static void __sdt_free(const struct cpumask *cpu_map) struct sched_domain_topology_level *tl; int j; - for (tl = sched_domain_topology; tl->init; tl++) { + for_each_sd_topology(tl) { struct sd_data *sdd = &tl->data; for_each_cpu(j, cpu_map) { @@ -8673,24 +6860,27 @@ static void __sdt_free(const struct cpumask *cpu_map) if (sdd->sg) kfree(*per_cpu_ptr(sdd->sg, j)); - if (sdd->sgp) - kfree(*per_cpu_ptr(sdd->sgp, j)); + if (sdd->sgc) + kfree(*per_cpu_ptr(sdd->sgc, j)); + if (sdd->sge) + kfree(*per_cpu_ptr(sdd->sge, j)); } free_percpu(sdd->sd); sdd->sd = NULL; free_percpu(sdd->sg); sdd->sg = NULL; - free_percpu(sdd->sgp); - sdd->sgp = NULL; + free_percpu(sdd->sgc); + sdd->sgc = NULL; + free_percpu(sdd->sge); + sdd->sge = NULL; } } struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl, - struct s_data *d, const struct cpumask *cpu_map, - struct sched_domain_attr *attr, struct sched_domain *child, - int cpu) + const struct cpumask *cpu_map, struct sched_domain_attr *attr, + struct sched_domain *child, int cpu) { - struct sched_domain *sd = tl->init(tl, cpu); + struct sched_domain *sd = sd_init(tl, cpu); if (!sd) return child; @@ -8699,8 +6889,22 @@ struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl, sd->level = child->level + 1; sched_domain_level_max = max(sched_domain_level_max, sd->level); child->parent = sd; + sd->child = child; + + if (!cpumask_subset(sched_domain_span(child), + sched_domain_span(sd))) { + pr_err("BUG: arch topology borken\n"); +#ifdef CONFIG_SCHED_DEBUG + pr_err(" the %s domain not a subset of the %s domain\n", + child->name, sd->name); +#endif + /* Fixup, ensure @sd has at least @child cpus. */ + cpumask_or(sched_domain_span(sd), + sched_domain_span(sd), + sched_domain_span(child)); + } + } - sd->child = child; set_domain_attribute(sd, attr); return sd; @@ -8713,9 +6917,10 @@ struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl, static int build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr) { - enum s_alloc alloc_state = sa_none; + enum s_alloc alloc_state; struct sched_domain *sd; struct s_data d; + struct rq *rq; int i, ret = -ENOMEM; alloc_state = __visit_domain_allocation_hell(&d, cpu_map); @@ -8727,18 +6932,13 @@ static int build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_topology_level *tl; sd = NULL; - for (tl = sched_domain_topology; tl->init; tl++) { - sd = build_sched_domain(tl, &d, cpu_map, attr, sd, i); + for_each_sd_topology(tl) { + sd = build_sched_domain(tl, cpu_map, attr, sd, i); + if (tl == sched_domain_topology) + *per_cpu_ptr(d.sd, i) = sd; if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP)) sd->flags |= SD_OVERLAP; - if (cpumask_equal(cpu_map, sched_domain_span(sd))) - break; } - - while (sd->child) - sd = sd->child; - - *per_cpu_ptr(d.sd, i) = sd; } /* Build the groups for the domains */ @@ -8755,25 +6955,35 @@ static int build_sched_domains(const struct cpumask *cpu_map, } } - /* Calculate CPU power for physical packages and nodes */ + /* Calculate CPU capacity for physical packages and nodes */ for (i = nr_cpumask_bits-1; i >= 0; i--) { + struct sched_domain_topology_level *tl = sched_domain_topology; + if (!cpumask_test_cpu(i, cpu_map)) continue; - for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { + for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent, tl++) { + init_sched_energy(i, sd, tl); claim_allocations(i, sd); - init_sched_groups_power(i, sd); + init_sched_groups_capacity(i, sd); } } /* Attach the domains */ rcu_read_lock(); for_each_cpu(i, cpu_map) { + rq = cpu_rq(i); sd = *per_cpu_ptr(d.sd, i); cpu_attach_domain(sd, d.rd, i); + + if (rq->cpu_capacity_orig > rq->rd->max_cpu_capacity) + rq->rd->max_cpu_capacity = rq->cpu_capacity_orig; } rcu_read_unlock(); + rq = cpu_rq(cpumask_first(cpu_map)); + pr_info("Max cpu capacity: %lu\n", rq->rd->max_cpu_capacity); + ret = 0; error: __free_domain_allocs(&d, alloc_state, cpu_map); @@ -8797,7 +7007,7 @@ static cpumask_var_t fallback_doms; * cpu core maps. It is supposed to return 1 if the topology changed * or 0 if it stayed the same. */ -int __attribute__((weak)) arch_update_cpu_topology(void) +int __weak arch_update_cpu_topology(void) { return 0; } @@ -8933,8 +7143,9 @@ void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], ; } + n = ndoms_cur; if (doms_new == NULL) { - ndoms_cur = 0; + n = 0; doms_new = &fallback_doms; cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map); WARN_ON_ONCE(dattr_new); @@ -8942,7 +7153,7 @@ void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], /* Build new domains */ for (i = 0; i < ndoms_new; i++) { - for (j = 0; j < ndoms_cur && !new_topology; j++) { + for (j = 0; j < n && !new_topology; j++) { if (cpumask_equal(doms_new[i], doms_cur[j]) && dattrs_equal(dattr_new, i, dattr_cur, j)) goto match2; @@ -9032,7 +7243,6 @@ void __init sched_init_smp(void) { cpumask_var_t non_isolated_cpus; - init_cpu_efficiency(); alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL); alloc_cpumask_var(&fallback_doms, GFP_KERNEL); @@ -9063,6 +7273,7 @@ void __init sched_init_smp(void) free_cpumask_var(non_isolated_cpus); init_sched_rt_class(); + init_sched_dl_class(); } #else void __init sched_init_smp(void) @@ -9096,11 +7307,6 @@ void __init sched_init(void) int i, j; unsigned long alloc_size = 0, ptr; - if (sched_enable_hmp) - pr_info("HMP scheduling enabled.\n"); - - BUG_ON(num_possible_cpus() > BITS_PER_LONG); - #ifdef CONFIG_FAIR_GROUP_SCHED alloc_size += 2 * nr_cpu_ids * sizeof(void **); #endif @@ -9137,13 +7343,15 @@ void __init sched_init(void) #endif /* CONFIG_CPUMASK_OFFSTACK */ } + init_rt_bandwidth(&def_rt_bandwidth, + global_rt_period(), global_rt_runtime()); + init_dl_bandwidth(&def_dl_bandwidth, + global_rt_period(), global_rt_runtime()); + #ifdef CONFIG_SMP init_defrootdomain(); #endif - init_rt_bandwidth(&def_rt_bandwidth, - global_rt_period(), global_rt_runtime()); - #ifdef CONFIG_RT_GROUP_SCHED init_rt_bandwidth(&root_task_group.rt_bandwidth, global_rt_period(), global_rt_runtime()); @@ -9160,7 +7368,6 @@ void __init sched_init(void) for_each_possible_cpu(i) { struct rq *rq; - per_cpu(dptr, i) = per_cpu(dbuf, i); rq = cpu_rq(i); raw_spin_lock_init(&rq->lock); rq->nr_running = 0; @@ -9168,6 +7375,7 @@ void __init sched_init(void) rq->calc_load_update = jiffies + LOAD_FREQ; init_cfs_rq(&rq->cfs); init_rt_rq(&rq->rt, rq); + init_dl_rq(&rq->dl, rq); #ifdef CONFIG_FAIR_GROUP_SCHED root_task_group.shares = ROOT_TASK_GROUP_LOAD; INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); @@ -9196,7 +7404,6 @@ void __init sched_init(void) rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; #ifdef CONFIG_RT_GROUP_SCHED - INIT_LIST_HEAD(&rq->leaf_rt_rq_list); init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); #endif @@ -9208,46 +7415,16 @@ void __init sched_init(void) #ifdef CONFIG_SMP rq->sd = NULL; rq->rd = NULL; - rq->cpu_power = SCHED_POWER_SCALE; + rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE; rq->post_schedule = 0; rq->active_balance = 0; rq->next_balance = jiffies; rq->push_cpu = 0; - rq->push_task = NULL; rq->cpu = i; rq->online = 0; rq->idle_stamp = 0; rq->avg_idle = 2*sysctl_sched_migration_cost; - rq->cstate = 0; - rq->wakeup_latency = 0; - rq->wakeup_energy = 0; -#ifdef CONFIG_SCHED_HMP - cpumask_set_cpu(i, &rq->freq_domain_cpumask); - rq->cur_freq = 1; - rq->max_freq = 1; - rq->min_freq = 1; - rq->max_possible_freq = 1; - rq->max_possible_capacity = 0; - rq->hmp_stats.cumulative_runnable_avg = 0; - rq->efficiency = 1024; - rq->capacity = 1024; - rq->load_scale_factor = 1024; - rq->window_start = 0; - rq->hmp_stats.nr_small_tasks = rq->hmp_stats.nr_big_tasks = 0; - rq->hmp_flags = 0; - rq->mostly_idle_load = pct_to_real(20); - rq->mostly_idle_nr_run = 3; - rq->mostly_idle_freq = 0; - rq->cur_irqload = 0; - rq->avg_irqload = 0; - rq->irqload_ts = 0; - rq->prefer_idle = 1; -#ifdef CONFIG_SCHED_FREQ_INPUT - rq->old_busy_time = 0; - rq->curr_runnable_sum = rq->prev_runnable_sum = 0; - rq->notifier_sent = 0; -#endif -#endif + rq->max_idle_balance_cost = sysctl_sched_migration_cost; INIT_LIST_HEAD(&rq->cfs_tasks); @@ -9263,24 +7440,23 @@ void __init sched_init(void) atomic_set(&rq->nr_iowait, 0); } - set_hmp_defaults(); - set_load_weight(&init_task); #ifdef CONFIG_PREEMPT_NOTIFIERS INIT_HLIST_HEAD(&init_task.preempt_notifiers); #endif -#ifdef CONFIG_RT_MUTEXES - plist_head_init(&init_task.pi_waiters); -#endif - /* * The boot idle thread does lazy MMU switching as well: */ atomic_inc(&init_mm.mm_count); enter_lazy_tlb(&init_mm, current); + /* + * During early bootup we pretend to be a normal task: + */ + current->sched_class = &fair_sched_class; + /* * Make us the idle thread. Technically, schedule() should not be * called from this thread, however somewhere below it might be, @@ -9291,17 +7467,13 @@ void __init sched_init(void) calc_load_update = jiffies + LOAD_FREQ; - /* - * During early bootup we pretend to be a normal task: - */ - current->sched_class = &fair_sched_class; - #ifdef CONFIG_SMP zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT); /* May be allocated at isolcpus cmdline parse time */ if (cpu_isolated_map == NULL) zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT); idle_thread_set_boot_cpu(); + set_cpu_rq_start_time(); #endif init_sched_fair_class(); @@ -9329,8 +7501,8 @@ void __might_sleep(const char *file, int line, int preempt_offset) static unsigned long prev_jiffy; /* ratelimiting */ rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */ - if ((preempt_count_equals(preempt_offset) && !irqs_disabled()) || - oops_in_progress) + if ((preempt_count_equals(preempt_offset) && !irqs_disabled() && + !is_idle_task(current)) || oops_in_progress) return; if (system_state != SYSTEM_RUNNING && (!__might_sleep_init_called || system_state != SYSTEM_BOOTING)) @@ -9350,6 +7522,13 @@ void __might_sleep(const char *file, int line, int preempt_offset) debug_show_held_locks(current); if (irqs_disabled()) print_irqtrace_events(current); +#ifdef CONFIG_DEBUG_PREEMPT + if (!preempt_count_equals(preempt_offset)) { + pr_err("Preemption disabled at:"); + print_ip_sym(current->preempt_disable_ip); + pr_cont("\n"); + } +#endif dump_stack(); } EXPORT_SYMBOL(__might_sleep); @@ -9363,15 +7542,15 @@ static void normalize_task(struct rq *rq, struct task_struct *p) .sched_policy = SCHED_NORMAL, }; int old_prio = p->prio; - int on_rq; + int queued; - on_rq = p->on_rq; - if (on_rq) + queued = task_on_rq_queued(p); + if (queued) dequeue_task(rq, p, 0); - __setscheduler(rq, p, &attr); - if (on_rq) { + __setscheduler(rq, p, &attr, false); + if (queued) { enqueue_task(rq, p, 0); - resched_task(rq->curr); + resched_curr(rq); } check_class_changed(rq, p, prev_class, old_prio); @@ -9383,12 +7562,12 @@ void normalize_rt_tasks(void) unsigned long flags; struct rq *rq; - read_lock_irqsave(&tasklist_lock, flags); - do_each_thread(g, p) { + read_lock(&tasklist_lock); + for_each_process_thread(g, p) { /* * Only normalize user tasks: */ - if (!p->mm) + if (p->flags & PF_KTHREAD) continue; p->se.exec_start = 0; @@ -9398,26 +7577,21 @@ void normalize_rt_tasks(void) p->se.statistics.block_start = 0; #endif - if (!rt_task(p)) { + if (!dl_task(p) && !rt_task(p)) { /* * Renice negative nice level userspace * tasks back to 0: */ - if (TASK_NICE(p) < 0 && p->mm) + if (task_nice(p) < 0) set_user_nice(p, 0); continue; } - raw_spin_lock(&p->pi_lock); - rq = __task_rq_lock(p); - + rq = task_rq_lock(p, &flags); normalize_task(rq, p); - - __task_rq_unlock(rq); - raw_spin_unlock(&p->pi_lock); - } while_each_thread(g, p); - - read_unlock_irqrestore(&tasklist_lock, flags); + task_rq_unlock(rq, p, &flags); + } + read_unlock(&tasklist_lock); } #endif /* CONFIG_MAGIC_SYSRQ */ @@ -9438,6 +7612,8 @@ void normalize_rt_tasks(void) * @cpu: the processor in question. * * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! + * + * Return: The current task for @cpu. */ struct task_struct *curr_task(int cpu) { @@ -9555,20 +7731,25 @@ void sched_offline_group(struct task_group *tg) void sched_move_task(struct task_struct *tsk) { struct task_group *tg; - int on_rq, running; + int queued, running; unsigned long flags; struct rq *rq; rq = task_rq_lock(tsk, &flags); running = task_current(rq, tsk); - on_rq = tsk->on_rq; + queued = task_on_rq_queued(tsk); - if (on_rq) + if (queued) dequeue_task(rq, tsk, 0); if (unlikely(running)) - tsk->sched_class->put_prev_task(rq, tsk); + put_prev_task(rq, tsk); + /* + * All callers are synchronized by task_rq_lock(); we do not use RCU + * which is pointless here. Thus, we pass "true" to task_css_check() + * to prevent lockdep warnings. + */ tg = container_of(task_subsys_state_check(tsk, cpu_cgroup_subsys_id, lockdep_is_held(&tsk->sighand->siglock)), struct task_group, css); @@ -9577,30 +7758,20 @@ void sched_move_task(struct task_struct *tsk) #ifdef CONFIG_FAIR_GROUP_SCHED if (tsk->sched_class->task_move_group) - tsk->sched_class->task_move_group(tsk, on_rq); + tsk->sched_class->task_move_group(tsk, queued); else #endif set_task_rq(tsk, task_cpu(tsk)); if (unlikely(running)) tsk->sched_class->set_curr_task(rq); - if (on_rq) + if (queued) enqueue_task(rq, tsk, 0); task_rq_unlock(rq, tsk, &flags); } #endif /* CONFIG_CGROUP_SCHED */ -#if defined(CONFIG_RT_GROUP_SCHED) || defined(CONFIG_CFS_BANDWIDTH) -static unsigned long to_ratio(u64 period, u64 runtime) -{ - if (runtime == RUNTIME_INF) - return 1ULL << 20; - - return div64_u64(runtime << 20, period); -} -#endif - #ifdef CONFIG_RT_GROUP_SCHED /* * Ensure that the real time constraints are schedulable. @@ -9612,10 +7783,16 @@ static inline int tg_has_rt_tasks(struct task_group *tg) { struct task_struct *g, *p; - do_each_thread(g, p) { - if (rt_task(p) && task_rq(p)->rt.tg == tg) + /* + * Autogroups do not have RT tasks; see autogroup_create(). + */ + if (task_group_is_autogroup(tg)) + return 0; + + for_each_process_thread(g, p) { + if (rt_task(p) && task_group(p) == tg) return 1; - } while_each_thread(g, p); + } return 0; } @@ -9774,24 +7951,13 @@ static long sched_group_rt_period(struct task_group *tg) do_div(rt_period_us, NSEC_PER_USEC); return rt_period_us; } +#endif /* CONFIG_RT_GROUP_SCHED */ +#ifdef CONFIG_RT_GROUP_SCHED static int sched_rt_global_constraints(void) { - u64 runtime, period; int ret = 0; - if (sysctl_sched_rt_period <= 0) - return -EINVAL; - - runtime = global_rt_runtime(); - period = global_rt_period(); - - /* - * Sanity check on the sysctl variables. - */ - if (runtime > period && runtime != RUNTIME_INF) - return -EINVAL; - mutex_lock(&rt_constraints_mutex); read_lock(&tasklist_lock); ret = __rt_schedulable(NULL, 0, 0); @@ -9814,78 +7980,160 @@ static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk) static int sched_rt_global_constraints(void) { unsigned long flags; - int i; + int i, ret = 0; - if (sysctl_sched_rt_period <= 0) - return -EINVAL; + raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); + for_each_possible_cpu(i) { + struct rt_rq *rt_rq = &cpu_rq(i)->rt; + + raw_spin_lock(&rt_rq->rt_runtime_lock); + rt_rq->rt_runtime = global_rt_runtime(); + raw_spin_unlock(&rt_rq->rt_runtime_lock); + } + raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); + + return ret; +} +#endif /* CONFIG_RT_GROUP_SCHED */ + +static int sched_dl_global_constraints(void) +{ + u64 runtime = global_rt_runtime(); + u64 period = global_rt_period(); + u64 new_bw = to_ratio(period, runtime); + struct dl_bw *dl_b; + int cpu, ret = 0; + unsigned long flags; + + /* + * Here we want to check the bandwidth not being set to some + * value smaller than the currently allocated bandwidth in + * any of the root_domains. + * + * FIXME: Cycling on all the CPUs is overdoing, but simpler than + * cycling on root_domains... Discussion on different/better + * solutions is welcome! + */ + for_each_possible_cpu(cpu) { + rcu_read_lock_sched(); + dl_b = dl_bw_of(cpu); + + raw_spin_lock_irqsave(&dl_b->lock, flags); + if (new_bw < dl_b->total_bw) + ret = -EBUSY; + raw_spin_unlock_irqrestore(&dl_b->lock, flags); + + rcu_read_unlock_sched(); + + if (ret) + break; + } + + return ret; +} + +static void sched_dl_do_global(void) +{ + u64 new_bw = -1; + struct dl_bw *dl_b; + int cpu; + unsigned long flags; + + def_dl_bandwidth.dl_period = global_rt_period(); + def_dl_bandwidth.dl_runtime = global_rt_runtime(); + + if (global_rt_runtime() != RUNTIME_INF) + new_bw = to_ratio(global_rt_period(), global_rt_runtime()); /* - * There's always some RT tasks in the root group - * -- migration, kstopmachine etc.. + * FIXME: As above... */ - if (sysctl_sched_rt_runtime == 0) - return -EBUSY; + for_each_possible_cpu(cpu) { + rcu_read_lock_sched(); + dl_b = dl_bw_of(cpu); - raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); - for_each_possible_cpu(i) { - struct rt_rq *rt_rq = &cpu_rq(i)->rt; + raw_spin_lock_irqsave(&dl_b->lock, flags); + dl_b->bw = new_bw; + raw_spin_unlock_irqrestore(&dl_b->lock, flags); - raw_spin_lock(&rt_rq->rt_runtime_lock); - rt_rq->rt_runtime = global_rt_runtime(); - raw_spin_unlock(&rt_rq->rt_runtime_lock); + rcu_read_unlock_sched(); } - raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); +} + +static int sched_rt_global_validate(void) +{ + if (sysctl_sched_rt_period <= 0) + return -EINVAL; + + if ((sysctl_sched_rt_runtime != RUNTIME_INF) && + (sysctl_sched_rt_runtime > sysctl_sched_rt_period)) + return -EINVAL; return 0; } -#endif /* CONFIG_RT_GROUP_SCHED */ -int sched_rr_handler(struct ctl_table *table, int write, +static void sched_rt_do_global(void) +{ + def_rt_bandwidth.rt_runtime = global_rt_runtime(); + def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period()); +} + +int sched_rt_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { - int ret; + int old_period, old_runtime; static DEFINE_MUTEX(mutex); + int ret; mutex_lock(&mutex); + old_period = sysctl_sched_rt_period; + old_runtime = sysctl_sched_rt_runtime; + ret = proc_dointvec(table, write, buffer, lenp, ppos); - /* make sure that internally we keep jiffies */ - /* also, writing zero resets timeslice to default */ + if (!ret && write) { - sched_rr_timeslice = sched_rr_timeslice <= 0 ? - RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice); + ret = sched_rt_global_validate(); + if (ret) + goto undo; + + ret = sched_rt_global_constraints(); + if (ret) + goto undo; + + ret = sched_dl_global_constraints(); + if (ret) + goto undo; + + sched_rt_do_global(); + sched_dl_do_global(); + } + if (0) { +undo: + sysctl_sched_rt_period = old_period; + sysctl_sched_rt_runtime = old_runtime; } mutex_unlock(&mutex); + return ret; } -int sched_rt_handler(struct ctl_table *table, int write, +int sched_rr_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { int ret; - int old_period, old_runtime; static DEFINE_MUTEX(mutex); mutex_lock(&mutex); - old_period = sysctl_sched_rt_period; - old_runtime = sysctl_sched_rt_runtime; - ret = proc_dointvec(table, write, buffer, lenp, ppos); - + /* make sure that internally we keep jiffies */ + /* also, writing zero resets timeslice to default */ if (!ret && write) { - ret = sched_rt_global_constraints(); - if (ret) { - sysctl_sched_rt_period = old_period; - sysctl_sched_rt_runtime = old_runtime; - } else { - def_rt_bandwidth.rt_runtime = global_rt_runtime(); - def_rt_bandwidth.rt_period = - ns_to_ktime(global_rt_period()); - } + sched_rr_timeslice = sched_rr_timeslice <= 0 ? + RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice); } mutex_unlock(&mutex); - return ret; } @@ -9984,6 +8232,7 @@ cpu_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp, sched_move_task(task); } +#ifdef TJK_HMP static u64 cpu_notify_on_migrate_read_u64(struct cgroup *cgrp, struct cftype *cft) { @@ -10001,6 +8250,7 @@ static int cpu_notify_on_migrate_write_u64(struct cgroup *cgrp, return 0; } +#endif #ifdef CONFIG_SCHED_HMP @@ -10087,11 +8337,6 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) if (period > max_cfs_quota_period) return -EINVAL; - /* - * Prevent race between setting of cfs_rq->runtime_enabled and - * unthrottle_offline_cfs_rqs(). - */ - get_online_cpus(); mutex_lock(&cfs_constraints_mutex); ret = __cfs_schedulable(tg, period, quota); if (ret) @@ -10113,11 +8358,12 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) /* restart the period timer (if active) to handle new period expiry */ if (runtime_enabled && cfs_b->timer_active) { /* force a reprogram */ - __start_cfs_bandwidth(cfs_b, true); + cfs_b->timer_active = 0; + __start_cfs_bandwidth(cfs_b); } raw_spin_unlock_irq(&cfs_b->lock); - for_each_online_cpu(i) { + for_each_possible_cpu(i) { struct cfs_rq *cfs_rq = tg->cfs_rq[i]; struct rq *rq = cfs_rq->rq; @@ -10133,39 +8379,10 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) cfs_bandwidth_usage_dec(); out_unlock: mutex_unlock(&cfs_constraints_mutex); - put_online_cpus(); return ret; } -#ifdef VENDOR_EDIT -int tg_set_cfs_quota_per_task(struct task_group *tg, long cfs_quota_us) -{ - struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; - u64 quota, period; - - period = ktime_to_ns(tg->cfs_bandwidth.period); - quota = tg->cfs_bandwidth.quota; - - raw_spin_lock_irq(&cfs_b->lock); - if (cfs_quota_us > 0) - cfs_b->quota_per_task = (u64)cfs_quota_us * NSEC_PER_USEC; - raw_spin_unlock_irq(&cfs_b->lock); - - return tg_set_cfs_bandwidth(tg, period, quota); -} - -long tg_get_cfs_quota_per_task(struct task_group *tg) -{ - u64 quota_us; - - quota_us = tg->cfs_bandwidth.quota_per_task; - do_div(quota_us, NSEC_PER_USEC); - - return quota_us; -} -#endif - int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) { u64 quota, period; @@ -10212,19 +8429,6 @@ long tg_get_cfs_period(struct task_group *tg) return cfs_period_us; } -#ifdef VENDOR_EDIT -static s64 cpu_cfs_quota_per_task_read_s64(struct cgroup *cgrp, struct cftype *cft) -{ - return tg_get_cfs_quota_per_task(cgroup_tg(cgrp)); -} - -static int cpu_cfs_quota_per_task_write_s64(struct cgroup *cgrp, struct cftype *cftype, - s64 cfs_quota_us) -{ - return tg_set_cfs_quota_per_task(cgroup_tg(cgrp), cfs_quota_us); -} -#endif - static s64 cpu_cfs_quota_read_s64(struct cgroup *cgrp, struct cftype *cft) { return tg_get_cfs_quota(cgroup_tg(cgrp)); @@ -10365,11 +8569,13 @@ static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft) #endif /* CONFIG_RT_GROUP_SCHED */ static struct cftype cpu_files[] = { +#ifdef TJK_HMP { .name = "notify_on_migrate", .read_u64 = cpu_notify_on_migrate_read_u64, .write_u64 = cpu_notify_on_migrate_write_u64, }, +#endif #ifdef CONFIG_SCHED_HMP { .name = "upmigrate_discourage", @@ -10385,13 +8591,6 @@ static struct cftype cpu_files[] = { }, #endif #ifdef CONFIG_CFS_BANDWIDTH -#ifdef VENDOR_EDIT - { - .name = "cfs_quota_us_per_task", - .read_s64 = cpu_cfs_quota_per_task_read_s64, - .write_s64 = cpu_cfs_quota_per_task_write_s64, - }, -#endif { .name = "cfs_quota_us", .read_s64 = cpu_cfs_quota_read_s64, @@ -10444,3 +8643,179 @@ void dump_cpu_task(int cpu) pr_info("Task dump for CPU %d:\n", cpu); sched_show_task(cpu_curr(cpu)); } + +// TJK: Below here: Added for 3.10 compatibility +static long __sched +sleep_on_common(wait_queue_head_t *q, int state, long timeout) +{ + unsigned long flags; + wait_queue_t wait; + + init_waitqueue_entry(&wait, current); + + __set_current_state(state); + + spin_lock_irqsave(&q->lock, flags); + __add_wait_queue(q, &wait); + spin_unlock(&q->lock); + timeout = schedule_timeout(timeout); + spin_lock_irq(&q->lock); + __remove_wait_queue(q, &wait); + spin_unlock_irqrestore(&q->lock, flags); + + return timeout; +} + +void __sched interruptible_sleep_on(wait_queue_head_t *q) +{ + sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); +} +EXPORT_SYMBOL(interruptible_sleep_on); + +long __sched +interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout) +{ + return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout); +} +EXPORT_SYMBOL(interruptible_sleep_on_timeout); + +void __sched sleep_on(wait_queue_head_t *q) +{ + sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); +} +EXPORT_SYMBOL(sleep_on); + +/* + * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just + * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve + * number) then we wake all the non-exclusive tasks and one exclusive task. + * + * There are circumstances in which we can try to wake a task which has already + * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns + * zero in this (rare) case, and we handle it by continuing to scan the queue. + */ +static void __wake_up_common(wait_queue_head_t *q, unsigned int mode, + int nr_exclusive, int wake_flags, void *key) +{ + wait_queue_t *curr, *next; + + list_for_each_entry_safe(curr, next, &q->task_list, task_list) { + unsigned flags = curr->flags; + + if (curr->func(curr, mode, wake_flags, key) && + (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive) + break; + } +} + +/** + * __wake_up - wake up threads blocked on a waitqueue. + * @q: the waitqueue + * @mode: which threads + * @nr_exclusive: how many wake-one or wake-many threads to wake up + * @key: is directly passed to the wakeup function + * + * It may be assumed that this function implies a write memory barrier before + * changing the task state if and only if any tasks are woken up. + */ +void __wake_up(wait_queue_head_t *q, unsigned int mode, + int nr_exclusive, void *key) +{ + unsigned long flags; + + spin_lock_irqsave(&q->lock, flags); + __wake_up_common(q, mode, nr_exclusive, 0, key); + spin_unlock_irqrestore(&q->lock, flags); +} +EXPORT_SYMBOL(__wake_up); + +/* + * Same as __wake_up but called with the spinlock in wait_queue_head_t held. + */ +void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr) +{ + __wake_up_common(q, mode, nr, 0, NULL); +} +EXPORT_SYMBOL_GPL(__wake_up_locked); + +void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key) +{ + __wake_up_common(q, mode, 1, 0, key); +} +EXPORT_SYMBOL_GPL(__wake_up_locked_key); + +/** + * __wake_up_sync_key - wake up threads blocked on a waitqueue. + * @q: the waitqueue + * @mode: which threads + * @nr_exclusive: how many wake-one or wake-many threads to wake up + * @key: opaque value to be passed to wakeup targets + * + * The sync wakeup differs that the waker knows that it will schedule + * away soon, so while the target thread will be woken up, it will not + * be migrated to another CPU - ie. the two threads are 'synchronized' + * with each other. This can prevent needless bouncing between CPUs. + * + * On UP it can prevent extra preemption. + * + * It may be assumed that this function implies a write memory barrier before + * changing the task state if and only if any tasks are woken up. + */ +void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, + int nr_exclusive, void *key) +{ + unsigned long flags; + int wake_flags = WF_SYNC; + + if (unlikely(!q)) + return; + + if (unlikely(!nr_exclusive)) + wake_flags = 0; + + spin_lock_irqsave(&q->lock, flags); + __wake_up_common(q, mode, nr_exclusive, wake_flags, key); + spin_unlock_irqrestore(&q->lock, flags); +} +EXPORT_SYMBOL_GPL(__wake_up_sync_key); + +/* + * __wake_up_sync - see __wake_up_sync_key() + */ +void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive) +{ + __wake_up_sync_key(q, mode, nr_exclusive, NULL); +} +EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ + +/* + * Note C-state for (idle) cpus. + * + * @cstate = cstate index, 0 -> active state + * @wakeup_energy = energy spent in waking up cpu + * @wakeup_latency = latency to wakeup from cstate + * + */ + +void +sched_set_cpu_cstate(int cpu, int cstate, struct cpuidle_state *cpuidle, + int wakeup_energy, int wakeup_latency) +{ + // TJK: Need to integrate QC cpuidle with EAS (doesn't use standard + // cpuidle framework) + struct rq *rq = cpu_rq(cpu); + + idle_set_state_idx(rq, cstate); + idle_set_state(rq, cpuidle); + + if ((cstate == 0 && wakeup_energy == 0 && wakeup_latency == 0) || + (cstate == 1 && cpu < 4 && wakeup_energy == 5800 && wakeup_latency == 50) || + (cstate == 1 && cpu >= 4 && wakeup_energy == 24120 && wakeup_latency == 60) || + (cstate == 3 && cpu >= 4 && wakeup_energy == 155775 && wakeup_latency == 475) || + (cstate == 4 && cpu < 4 && wakeup_energy == 46400 && wakeup_latency == 480) || + (cstate == 4 && cpu >= 4 && wakeup_energy == 192156 && wakeup_latency == 636)) { +// } else { +// pr_info("TJK: CPU%d ->cstate=%d nrg=%d latency=%d\n", cpu, cstate, wakeup_energy, wakeup_latency); + } +} + diff --git a/kernel/sched/cpuacct.c b/kernel/sched/cpuacct.c index dbb7e2cd95eba..f0186c764ee14 100644 --- a/kernel/sched/cpuacct.c +++ b/kernel/sched/cpuacct.c @@ -33,7 +33,11 @@ struct cpuacct { struct kernel_cpustat __percpu *cpustat; }; -/* return cpu accounting group corresponding to this container */ +static inline struct cpuacct *css_ca(struct cgroup_subsys_state *css) +{ + return css ? container_of(css, struct cpuacct, css) : NULL; +} + static inline struct cpuacct *cgroup_ca(struct cgroup *cgrp) { return container_of(cgroup_subsys_state(cgrp, cpuacct_subsys_id), @@ -43,8 +47,7 @@ static inline struct cpuacct *cgroup_ca(struct cgroup *cgrp) /* return cpu accounting group to which this task belongs */ static inline struct cpuacct *task_ca(struct task_struct *tsk) { - return container_of(task_subsys_state(tsk, cpuacct_subsys_id), - struct cpuacct, css); + return container_of(task_subsys_state(tsk, cpuacct_subsys_id), struct cpuacct, css); } static inline struct cpuacct *__parent_ca(struct cpuacct *ca) @@ -172,8 +175,7 @@ static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype, return err; } -static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft, - struct seq_file *m) +static int cpuacct_percpu_seq_show(struct cgroup *cgroup, struct cftype *cft, struct seq_file *m) { struct cpuacct *ca = cgroup_ca(cgroup); u64 percpu; @@ -229,7 +231,7 @@ static struct cftype files[] = { }, { .name = "usage_percpu", - .read_seq_string = cpuacct_percpu_seq_read, + .read_seq_string = cpuacct_percpu_seq_show, }, { .name = "stat", @@ -281,16 +283,16 @@ void cpuacct_account_field(struct task_struct *p, int index, u64 val) while (ca != &root_cpuacct) { kcpustat = this_cpu_ptr(ca->cpustat); kcpustat->cpustat[index] += val; - ca = __parent_ca(ca); + ca = parent_ca(ca); } rcu_read_unlock(); } struct cgroup_subsys cpuacct_subsys = { - .name = "cpuacct", + .name = "cpuacct", .css_alloc = cpuacct_css_alloc, .css_free = cpuacct_css_free, - .subsys_id = cpuacct_subsys_id, + .subsys_id = cpuacct_subsys_id, .base_cftypes = files, .early_init = 1, }; diff --git a/kernel/sched/cpudeadline.c b/kernel/sched/cpudeadline.c new file mode 100644 index 0000000000000..539ca3ce071b2 --- /dev/null +++ b/kernel/sched/cpudeadline.c @@ -0,0 +1,227 @@ +/* + * kernel/sched/cpudl.c + * + * Global CPU deadline management + * + * Author: Juri Lelli + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + */ + +#include +#include +#include +#include "cpudeadline.h" + +static inline int parent(int i) +{ + return (i - 1) >> 1; +} + +static inline int left_child(int i) +{ + return (i << 1) + 1; +} + +static inline int right_child(int i) +{ + return (i << 1) + 2; +} + +static inline int dl_time_before(u64 a, u64 b) +{ + return (s64)(a - b) < 0; +} + +static void cpudl_exchange(struct cpudl *cp, int a, int b) +{ + int cpu_a = cp->elements[a].cpu, cpu_b = cp->elements[b].cpu; + + swap(cp->elements[a].cpu, cp->elements[b].cpu); + swap(cp->elements[a].dl , cp->elements[b].dl ); + + swap(cp->elements[cpu_a].idx, cp->elements[cpu_b].idx); +} + +static void cpudl_heapify(struct cpudl *cp, int idx) +{ + int l, r, largest; + + /* adapted from lib/prio_heap.c */ + while(1) { + l = left_child(idx); + r = right_child(idx); + largest = idx; + + if ((l < cp->size) && dl_time_before(cp->elements[idx].dl, + cp->elements[l].dl)) + largest = l; + if ((r < cp->size) && dl_time_before(cp->elements[largest].dl, + cp->elements[r].dl)) + largest = r; + if (largest == idx) + break; + + /* Push idx down the heap one level and bump one up */ + cpudl_exchange(cp, largest, idx); + idx = largest; + } +} + +static void cpudl_change_key(struct cpudl *cp, int idx, u64 new_dl) +{ + WARN_ON(idx == IDX_INVALID || !cpu_present(idx)); + + if (dl_time_before(new_dl, cp->elements[idx].dl)) { + cp->elements[idx].dl = new_dl; + cpudl_heapify(cp, idx); + } else { + cp->elements[idx].dl = new_dl; + while (idx > 0 && dl_time_before(cp->elements[parent(idx)].dl, + cp->elements[idx].dl)) { + cpudl_exchange(cp, idx, parent(idx)); + idx = parent(idx); + } + } +} + +static inline int cpudl_maximum(struct cpudl *cp) +{ + return cp->elements[0].cpu; +} + +/* + * cpudl_find - find the best (later-dl) CPU in the system + * @cp: the cpudl max-heap context + * @p: the task + * @later_mask: a mask to fill in with the selected CPUs (or NULL) + * + * Returns: int - best CPU (heap maximum if suitable) + */ +int cpudl_find(struct cpudl *cp, struct task_struct *p, + struct cpumask *later_mask) +{ + int best_cpu = -1; + const struct sched_dl_entity *dl_se = &p->dl; + + if (later_mask && cpumask_and(later_mask, later_mask, cp->free_cpus)) { + best_cpu = cpumask_any(later_mask); + goto out; + } else if (cpumask_test_cpu(cpudl_maximum(cp), &p->cpus_allowed) && + dl_time_before(dl_se->deadline, cp->elements[0].dl)) { + best_cpu = cpudl_maximum(cp); + if (later_mask) + cpumask_set_cpu(best_cpu, later_mask); + } + +out: + WARN_ON(best_cpu != -1 && !cpu_present(best_cpu)); + + return best_cpu; +} + +/* + * cpudl_set - update the cpudl max-heap + * @cp: the cpudl max-heap context + * @cpu: the target cpu + * @dl: the new earliest deadline for this cpu + * + * Notes: assumes cpu_rq(cpu)->lock is locked + * + * Returns: (void) + */ +void cpudl_set(struct cpudl *cp, int cpu, u64 dl, int is_valid) +{ + int old_idx, new_cpu; + unsigned long flags; + + WARN_ON(!cpu_present(cpu)); + + raw_spin_lock_irqsave(&cp->lock, flags); + old_idx = cp->elements[cpu].idx; + if (!is_valid) { + /* remove item */ + if (old_idx == IDX_INVALID) { + /* + * Nothing to remove if old_idx was invalid. + * This could happen if a rq_offline_dl is + * called for a CPU without -dl tasks running. + */ + goto out; + } + new_cpu = cp->elements[cp->size - 1].cpu; + cp->elements[old_idx].dl = cp->elements[cp->size - 1].dl; + cp->elements[old_idx].cpu = new_cpu; + cp->size--; + cp->elements[new_cpu].idx = old_idx; + cp->elements[cpu].idx = IDX_INVALID; + while (old_idx > 0 && dl_time_before( + cp->elements[parent(old_idx)].dl, + cp->elements[old_idx].dl)) { + cpudl_exchange(cp, old_idx, parent(old_idx)); + old_idx = parent(old_idx); + } + cpumask_set_cpu(cpu, cp->free_cpus); + cpudl_heapify(cp, old_idx); + + goto out; + } + + if (old_idx == IDX_INVALID) { + cp->size++; + cp->elements[cp->size - 1].dl = 0; + cp->elements[cp->size - 1].cpu = cpu; + cp->elements[cpu].idx = cp->size - 1; + cpudl_change_key(cp, cp->size - 1, dl); + cpumask_clear_cpu(cpu, cp->free_cpus); + } else { + cpudl_change_key(cp, old_idx, dl); + } + +out: + raw_spin_unlock_irqrestore(&cp->lock, flags); +} + +/* + * cpudl_init - initialize the cpudl structure + * @cp: the cpudl max-heap context + */ +int cpudl_init(struct cpudl *cp) +{ + int i; + + memset(cp, 0, sizeof(*cp)); + raw_spin_lock_init(&cp->lock); + cp->size = 0; + + cp->elements = kcalloc(nr_cpu_ids, + sizeof(struct cpudl_item), + GFP_KERNEL); + if (!cp->elements) + return -ENOMEM; + + if (!alloc_cpumask_var(&cp->free_cpus, GFP_KERNEL)) { + kfree(cp->elements); + return -ENOMEM; + } + + for_each_possible_cpu(i) + cp->elements[i].idx = IDX_INVALID; + + cpumask_setall(cp->free_cpus); + + return 0; +} + +/* + * cpudl_cleanup - clean up the cpudl structure + * @cp: the cpudl max-heap context + */ +void cpudl_cleanup(struct cpudl *cp) +{ + free_cpumask_var(cp->free_cpus); + kfree(cp->elements); +} diff --git a/kernel/sched/cpudeadline.h b/kernel/sched/cpudeadline.h new file mode 100644 index 0000000000000..538c9796ad4a0 --- /dev/null +++ b/kernel/sched/cpudeadline.h @@ -0,0 +1,33 @@ +#ifndef _LINUX_CPUDL_H +#define _LINUX_CPUDL_H + +#include + +#define IDX_INVALID -1 + +struct cpudl_item { + u64 dl; + int cpu; + int idx; +}; + +struct cpudl { + raw_spinlock_t lock; + int size; + cpumask_var_t free_cpus; + struct cpudl_item *elements; +}; + + +#ifdef CONFIG_SMP +int cpudl_find(struct cpudl *cp, struct task_struct *p, + struct cpumask *later_mask); +void cpudl_set(struct cpudl *cp, int cpu, u64 dl, int is_valid); +int cpudl_init(struct cpudl *cp); +void cpudl_cleanup(struct cpudl *cp); +#else +#define cpudl_set(cp, cpu, dl) do { } while (0) +#define cpudl_init() do { } while (0) +#endif /* CONFIG_SMP */ + +#endif /* _LINUX_CPUDL_H */ diff --git a/kernel/sched/cpufreq_sched.c b/kernel/sched/cpufreq_sched.c new file mode 100644 index 0000000000000..1065ee292b743 --- /dev/null +++ b/kernel/sched/cpufreq_sched.c @@ -0,0 +1,520 @@ +/* + * Copyright (C) 2015 Michael Turquette + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include + +#include "sched.h" + +#define THROTTLE_DOWN_NSEC 50000000 /* 50ms default */ +#define THROTTLE_UP_NSEC 500000 /* 500us default */ + +static DEFINE_PER_CPU(unsigned long, pcpu_capacity); +static DEFINE_PER_CPU(struct cpufreq_policy *, pcpu_policy); +static DEFINE_PER_CPU(int, governor_started); + +/** + * gov_data - per-policy data internal to the governor + * @up_throttle: next throttling period expiry if increasing OPP + * @down_throttle: next throttling period expiry if decreasing OPP + * @up_throttle_nsec: throttle period length in nanoseconds if increasing OPP + * @down_throttle_nsec: throttle period length in nanoseconds if decreasing OPP + * @task: worker thread for dvfs transition that may block/sleep + * @irq_work: callback used to wake up worker thread + * @requested_freq: last frequency requested by the sched governor + * + * struct gov_data is the per-policy cpufreq_sched-specific data structure. A + * per-policy instance of it is created when the cpufreq_sched governor receives + * the CPUFREQ_GOV_START condition and a pointer to it exists in the gov_data + * member of struct cpufreq_policy. + * + * Readers of this data must call down_read(policy->rwsem). Writers must + * call down_write(policy->rwsem). + */ +struct gov_data { + ktime_t up_throttle; + ktime_t down_throttle; + unsigned int up_throttle_nsec; + unsigned int down_throttle_nsec; + struct cpufreq_policy *policy; + unsigned int freq; + bool change_pending; + struct list_head gov_list; + int max; +}; + + /* worker thread for dvfs transition that may block/sleep */ +static struct task_struct *freq_change_task; +static LIST_HEAD(sched_gov_list); +static struct mutex gov_list_lock; + +/* irq_work: callback used to wake up worker thread */ +static struct irq_work irq_work; + +static void cpufreq_sched_try_driver_target(struct cpufreq_policy *policy, unsigned int freq) +{ + struct gov_data *gd; + + /* avoid race with cpufreq_sched_stop */ + if (!down_write_trylock(&policy->rwsem)) + return; + + gd = policy->governor_data; + + if (!gd) + return; + + __cpufreq_driver_target(policy, freq, CPUFREQ_RELATION_L); + + gd->up_throttle = ktime_add_ns(ktime_get(), gd->up_throttle_nsec); + gd->down_throttle = ktime_add_ns(ktime_get(), gd->down_throttle_nsec); + up_write(&policy->rwsem); +} + +/* + * we pass in struct cpufreq_policy. This is safe because changing out the + * policy requires a call to __cpufreq_governor(policy, CPUFREQ_GOV_STOP), + * which tears down all of the data structures and __cpufreq_governor(policy, + * CPUFREQ_GOV_START) will do a full rebuild, including this kthread with the + * new policy pointer + */ +static int cpufreq_sched_thread(void *data) +{ + struct sched_param param; + int ret; + + param.sched_priority = 50; + ret = sched_setscheduler_nocheck(current, SCHED_FIFO, ¶m); + if (ret) { + pr_warn("%s: failed to set SCHED_FIFO\n", __func__); + do_exit(-EINVAL); + } else { + pr_debug("%s: kthread (%d) set to SCHED_FIFO\n", + __func__, current->pid); + } + + /* main loop of the frequency change kthread */ + do { + struct cpufreq_policy *policy = NULL; + unsigned int freq = 0; + struct gov_data *gd = NULL; + mutex_lock(&gov_list_lock); + list_for_each_entry(gd, &sched_gov_list, gov_list) { + if (gd->change_pending) { + gd->change_pending = false; + freq = gd->freq; + policy = gd->policy; + break; + } + } + mutex_unlock(&gov_list_lock); + + if (!policy) { + set_current_state(TASK_INTERRUPTIBLE); + schedule(); + } else { + cpufreq_sched_try_driver_target(policy, freq); + } + + if (kthread_should_stop()) + break; + + } while (!kthread_should_stop()); + + do_exit(0); +} + +static void cpufreq_sched_irq_work(struct irq_work *irq_work) +{ + wake_up_process(freq_change_task); +} + +/** + * cpufreq_sched_set_capacity - interface to scheduler for changing capacity values + * @cpu: cpu whose capacity utilization has recently changed + * @capacity: the new capacity requested by cpu + * + * cpufreq_sched_sched_capacity is an interface exposed to the scheduler so + * that the scheduler may inform the governor of updates to capacity + * utilization and make changes to cpu frequency. Currently this interface is + * designed around PELT values in CFS. It can be expanded to other scheduling + * classes in the future if needed. + * + * cpufreq_sched_set_capacity raises an IPI. The irq_work handler for that IPI + * wakes up the thread that does the actual work, cpufreq_sched_thread. + * + * This functions bails out early if either condition is true: + * 1) this cpu did not the new maximum capacity for its frequency domain + * 2) no change in cpu frequency is necessary to meet the new capacity request + */ + +void cpufreq_sched_set_cap(int cpu, unsigned long capacity) +{ + unsigned int freq_new, cpu_tmp; + struct cpufreq_policy *policy; + struct gov_data *gd; + unsigned long capacity_max = 0; + + if (!per_cpu(governor_started, cpu)) + return; + + /* update per-cpu capacity request */ + per_cpu(pcpu_capacity, cpu) = capacity; + + policy = cpufreq_cpu_get(cpu); + if (IS_ERR_OR_NULL(policy)) { + return; + } + + if (!policy->governor_data) + goto out; + + gd = policy->governor_data; + + ktime_t throttle = gd->freq < policy->cur ? + gd->down_throttle : gd->up_throttle; + + /* bail early if we are throttled */ + if (ktime_compare(ktime_get(), throttle) < 0) + goto out; + + /* find max capacity requested by cpus in this policy */ + for_each_cpu(cpu_tmp, policy->cpus) + capacity_max = max(capacity_max, per_cpu(pcpu_capacity, cpu_tmp)); + + /* + * We only change frequency if this cpu's capacity request represents a + * new max. If another cpu has requested a capacity greater than the + * previous max then we rely on that cpu to hit this code path and make + * the change. IOW, the cpu with the new max capacity is responsible + * for setting the new capacity/frequency. + * + * If this cpu is not the new maximum then bail + */ + if (capacity_max > capacity) + goto out; + + /* Convert the new maximum capacity request into a cpu frequency */ + freq_new = (capacity * gd->max) / capacity_orig_of(cpu); + + if (freq_new > policy->max) + freq_new = policy->max; + + if (freq_new < policy->min) + freq_new = policy->min; + + /* No change in frequency? Bail and return current capacity. */ + if (freq_new == policy->cur) + goto out; + + /* store the new frequency and perform the transition */ + gd->freq = freq_new; + gd->change_pending = true; + + irq_work_queue(&irq_work); + +out: + cpufreq_cpu_put(policy); + return; +} + +/** + * cpufreq_sched_reset_capacity - interface to scheduler for resetting capacity + * requests + * @cpu: cpu whose capacity request has to be reset + * + * This _wont trigger_ any capacity update. + */ +void cpufreq_sched_reset_cap(int cpu) +{ + per_cpu(pcpu_capacity, cpu) = 0; +} + +static inline void set_sched_energy_freq(void) +{ + // TJK: must always bump the key so we can handle + // 1 cluster going online/offline + static_key_slow_inc(&__sched_energy_freq); +} + +static inline void clear_sched_energy_freq(void) +{ + // TJK: must always bump the key so we can handle + // 1 cluster going online/offline + static_key_slow_dec(&__sched_energy_freq); +} + +static int cpufreq_sched_policy_start(struct cpufreq_policy *policy) +{ + int cpu; + + /* initialize per-cpu data */ + for_each_cpu(cpu, policy->cpus) { + pr_info("%s: start CPU%d\n", __func__, cpu); + per_cpu(pcpu_capacity, cpu) = 0; + per_cpu(pcpu_policy, cpu) = policy; + } + return 0; +} + +static int cpufreq_sched_policy_stop(struct cpufreq_policy *policy) +{ + int cpu; + if (!policy) { + return 0; + } + for_each_cpu(cpu, policy->cpus) { + pr_info("%s: stop CPU%d\n", __func__, cpu); + } + /* + * Nothing to do. The per_cpu fields will be re-initialized + * on the next START + */ + return 0; +} + +static struct attribute_group sched_attr_group_gov_pol; +static struct attribute_group *get_sysfs_attr(void) +{ + return &sched_attr_group_gov_pol; +} + +static int cpufreq_sched_policy_init(struct cpufreq_policy *policy) +{ + int rc; + struct gov_data *gd; + int cpu = cpumask_first(policy->cpus); + + if (cpu >= nr_cpu_ids) { + return -EINVAL; + } + WARN_ON(policy->governor_data); + /* prepare per-policy private data */ + gd = kzalloc(sizeof(*gd), GFP_KERNEL); + pr_info("%s: init CPU%d cluster\n", __func__, cpu); + + if (!gd) { + pr_debug("%s: failed to allocate private data\n", __func__); + return -ENOMEM; + } + rc = sysfs_create_group(get_governor_parent_kobj(policy), get_sysfs_attr()); + if (rc) { + pr_err("%s: couldn't create sysfs attributes: %d\n", __func__, rc); + goto err; + } + + gd->up_throttle_nsec = policy->cpuinfo.transition_latency ? + policy->cpuinfo.transition_latency : + THROTTLE_UP_NSEC; + gd->down_throttle_nsec = THROTTLE_DOWN_NSEC; + pr_debug("%s: throttle threshold = %u [ns]\n", + __func__, gd->up_throttle_nsec); + + policy->governor_data = gd; + gd->policy = policy; + gd->max = policy->max; + + mutex_lock(&gov_list_lock); + list_add_tail(&gd->gov_list, &sched_gov_list); + mutex_unlock(&gov_list_lock); + + set_sched_energy_freq(); + + for_each_cpu(cpu, policy->cpus) + per_cpu(governor_started, cpu) = 1; + + return 0; + +err: + kfree(gd); + return -ENOMEM; +} + +static int cpufreq_sched_policy_exit(struct cpufreq_policy *policy) +{ + struct gov_data *gd = policy->governor_data; + int cpu = cpumask_first(policy->cpus); + pr_info("%s: exit CPU%d cluster\n", __func__, cpu); + WARN_ON(!policy->governor_data); + + for_each_cpu(cpu, policy->cpus) + per_cpu(governor_started, cpu) = 0; + + clear_sched_energy_freq(); + + sysfs_remove_group(get_governor_parent_kobj(policy), get_sysfs_attr()); + + policy->governor_data = NULL; + mutex_lock(&gov_list_lock); + list_del(&gd->gov_list); + mutex_unlock(&gov_list_lock); + + sysfs_remove_group(get_governor_parent_kobj(policy), get_sysfs_attr()); + + /* FIXME replace with devm counterparts? */ + kfree(gd); + return 0; +} + +static int cpufreq_sched_setup(struct cpufreq_policy *policy, unsigned int event) +{ + struct gov_data *gd; + switch (event) { + case CPUFREQ_GOV_START: + return cpufreq_sched_policy_start(policy); + case CPUFREQ_GOV_STOP: + return cpufreq_sched_policy_stop(policy); + case CPUFREQ_GOV_POLICY_INIT: + return cpufreq_sched_policy_init(policy); + case CPUFREQ_GOV_POLICY_EXIT: + return cpufreq_sched_policy_exit(policy); + case CPUFREQ_GOV_LIMITS: + mutex_lock(&gov_list_lock); + pr_debug("limit event for cpu %u: %u - %u kHz, currently %u kHz\n", + policy->cpu, policy->min, policy->max, + policy->cur); + /* + * Need to keep track of highest max frequency for + * capacity calculations + */ + gd = policy->governor_data; + if (gd->max < policy->max) + gd->max = policy->max; + + if (policy->max < policy->cur) + __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H); + else if (policy->min > policy->cur) + __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L); + mutex_unlock(&gov_list_lock); + break; + } + return 0; +} + +/* Tunables */ +static ssize_t show_up_throttle_nsec(struct gov_data *gd, char *buf) +{ + return sprintf(buf, "%u\n", gd->up_throttle_nsec); +} + +static ssize_t store_up_throttle_nsec(struct gov_data *gd, + const char *buf, size_t count) +{ + int ret; + long unsigned int val; + + ret = kstrtoul(buf, 0, &val); + if (ret < 0) + return ret; + gd->up_throttle_nsec = val; + return count; +} + +static ssize_t show_down_throttle_nsec(struct gov_data *gd, char *buf) +{ + return sprintf(buf, "%u\n", gd->down_throttle_nsec); +} + +static ssize_t store_down_throttle_nsec(struct gov_data *gd, + const char *buf, size_t count) +{ + int ret; + long unsigned int val; + + ret = kstrtoul(buf, 0, &val); + if (ret < 0) + return ret; + gd->down_throttle_nsec = val; + return count; +} + +/* + * Create show/store routines + * - sys: One governor instance for complete SYSTEM + * - pol: One governor instance per struct cpufreq_policy + */ +#define show_gov_pol_sys(file_name) \ +static ssize_t show_##file_name##_gov_pol \ +(struct cpufreq_policy *policy, char *buf) \ +{ \ + return show_##file_name(policy->governor_data, buf); \ +} + +#define store_gov_pol_sys(file_name) \ +static ssize_t store_##file_name##_gov_pol \ +(struct cpufreq_policy *policy, const char *buf, size_t count) \ +{ \ + return store_##file_name(policy->governor_data, buf, count); \ +} + +#define gov_pol_attr_rw(_name) \ + static struct freq_attr _name##_gov_pol = \ + __ATTR(_name, 0644, show_##_name##_gov_pol, store_##_name##_gov_pol) + +#define show_store_gov_pol_sys(file_name) \ + show_gov_pol_sys(file_name); \ + store_gov_pol_sys(file_name) +#define tunable_handlers(file_name) \ + show_gov_pol_sys(file_name); \ + store_gov_pol_sys(file_name); \ + gov_pol_attr_rw(file_name) + +tunable_handlers(down_throttle_nsec); +tunable_handlers(up_throttle_nsec); + +/* Per policy governor instance */ +static struct attribute *sched_attributes_gov_pol[] = { + &up_throttle_nsec_gov_pol.attr, + &down_throttle_nsec_gov_pol.attr, + NULL, +}; + +static struct attribute_group sched_attr_group_gov_pol = { + .attrs = sched_attributes_gov_pol, + .name = "sched", +}; + +#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHED +static +#endif +struct cpufreq_governor cpufreq_gov_sched = { + .name = "sched", + .governor = cpufreq_sched_setup, + .owner = THIS_MODULE, +}; + +static int __init cpufreq_sched_init(void) +{ + int cpu; + + for_each_cpu(cpu, cpu_possible_mask) { + per_cpu(governor_started, cpu) = 0; + } + mutex_init(&gov_list_lock); + freq_change_task = kthread_run(cpufreq_sched_thread, NULL, "kcpufreq_sched_task"); + if (IS_ERR_OR_NULL(freq_change_task)) { + pr_err("%s: failed to create kcpufreq_sched_task thread\n", __func__); + return -ENOMEM; + } + init_irq_work(&irq_work, cpufreq_sched_irq_work); + return cpufreq_register_governor(&cpufreq_gov_sched); +} + +static void __exit cpufreq_sched_exit(void) +{ + kthread_stop(freq_change_task); + cpufreq_unregister_governor(&cpufreq_gov_sched); +} + +/* Try to make this the default governor */ +fs_initcall(cpufreq_sched_init); + +MODULE_LICENSE("GPL v2"); diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c index 61d9e72c9df96..981fcd7dc394e 100644 --- a/kernel/sched/cpupri.c +++ b/kernel/sched/cpupri.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "cpupri.h" /* Convert between a 140 based task->prio, and our 102 based cpupri */ @@ -62,7 +63,7 @@ static int convert_prio(int prio) * any discrepancies created by racing against the uncertainty of the current * priority configuration. * - * Returns: (int)bool - CPUs were found + * Return: (int)bool - CPUs were found */ int cpupri_find(struct cpupri *cp, struct task_struct *p, struct cpumask *lowest_mask) @@ -202,7 +203,7 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri) * cpupri_init - initialize the cpupri structure * @cp: The cpupri context * - * Returns: -ENOMEM if memory fails. + * Return: -ENOMEM on memory allocation failure. */ int cpupri_init(struct cpupri *cp) { @@ -218,8 +219,13 @@ int cpupri_init(struct cpupri *cp) goto cleanup; } + cp->cpu_to_pri = kcalloc(nr_cpu_ids, sizeof(int), GFP_KERNEL); + if (!cp->cpu_to_pri) + goto cleanup; + for_each_possible_cpu(i) cp->cpu_to_pri[i] = CPUPRI_INVALID; + return 0; cleanup: @@ -236,6 +242,7 @@ void cpupri_cleanup(struct cpupri *cp) { int i; + kfree(cp->cpu_to_pri); for (i = 0; i < CPUPRI_NR_PRIORITIES; i++) free_cpumask_var(cp->pri_to_cpu[i].mask); } diff --git a/kernel/sched/cpupri.h b/kernel/sched/cpupri.h index f6d756173491c..6b033347fdfd1 100644 --- a/kernel/sched/cpupri.h +++ b/kernel/sched/cpupri.h @@ -17,7 +17,7 @@ struct cpupri_vec { struct cpupri { struct cpupri_vec pri_to_cpu[CPUPRI_NR_PRIORITIES]; - int cpu_to_pri[NR_CPUS]; + int *cpu_to_pri; }; #ifdef CONFIG_SMP diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 199dda8db9a0e..481ded956affa 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -50,7 +50,6 @@ void irqtime_account_irq(struct task_struct *curr) unsigned long flags; s64 delta; int cpu; - u64 wallclock; if (!sched_clock_irqtime) return; @@ -58,8 +57,7 @@ void irqtime_account_irq(struct task_struct *curr) local_irq_save(flags); cpu = smp_processor_id(); - wallclock = sched_clock_cpu(cpu); - delta = wallclock - __this_cpu_read(irq_start_time); + delta = sched_clock_cpu(cpu) - __this_cpu_read(irq_start_time); __this_cpu_add(irq_start_time, delta); irq_time_write_begin(); @@ -69,14 +67,10 @@ void irqtime_account_irq(struct task_struct *curr) * in that case, so as not to confuse scheduler with a special task * that do not consume any time, but still wants to run. */ - if (hardirq_count()) { + if (hardirq_count()) __this_cpu_add(cpu_hardirq_time, delta); - sched_account_irqtime(cpu, curr, delta, wallclock); - } else if (in_serving_softirq() && curr != this_cpu_ksoftirqd()) { + else if (in_serving_softirq() && curr != this_cpu_ksoftirqd()) __this_cpu_add(cpu_softirq_time, delta); - sched_account_irqtime(cpu, curr, delta, wallclock); - } - irq_time_write_end(); local_irq_restore(flags); @@ -128,7 +122,7 @@ static inline void task_group_account_field(struct task_struct *p, int index, * is the only cgroup, then nothing else should be necessary. * */ - __get_cpu_var(kernel_cpustat).cpustat[index] += tmp; + __this_cpu_add(kernel_cpustat.cpustat[index], tmp); cpuacct_account_field(p, index, tmp); } @@ -149,7 +143,7 @@ void account_user_time(struct task_struct *p, cputime_t cputime, p->utimescaled += cputime_scaled; account_group_user_time(p, cputime); - index = (TASK_NICE(p) > 0) ? CPUTIME_NICE : CPUTIME_USER; + index = (task_nice(p) > 0) ? CPUTIME_NICE : CPUTIME_USER; /* Add user time to cpustat. */ task_group_account_field(p, index, (__force u64) cputime); @@ -179,7 +173,7 @@ static void account_guest_time(struct task_struct *p, cputime_t cputime, p->gtime += cputime; /* Add guest time to cpustat. */ - if (TASK_NICE(p) > 0) { + if (task_nice(p) > 0) { cpustat[CPUTIME_NICE] += (__force u64) cputime; cpustat[CPUTIME_GUEST_NICE] += (__force u64) cputime; } else { @@ -271,16 +265,22 @@ static __always_inline bool steal_account_process_tick(void) { #ifdef CONFIG_PARAVIRT if (static_key_false(¶virt_steal_enabled)) { - u64 steal, st = 0; + u64 steal; + cputime_t steal_ct; steal = paravirt_steal_clock(smp_processor_id()); steal -= this_rq()->prev_steal_time; - st = steal_ticks(steal); - this_rq()->prev_steal_time += st * TICK_NSEC; + /* + * cputime_t may be less precise than nsecs (eg: if it's + * based on jiffies). Lets cast the result to cputime + * granularity and account the rest on the next rounds. + */ + steal_ct = nsecs_to_cputime(steal); + this_rq()->prev_steal_time += cputime_to_nsecs(steal_ct); - account_steal_time(st); - return st; + account_steal_time(steal_ct); + return steal_ct; } #endif return false; @@ -391,11 +391,8 @@ static inline void irqtime_account_process_tick(struct task_struct *p, int user_ #ifdef CONFIG_VIRT_CPU_ACCOUNTING #ifndef __ARCH_HAS_VTIME_TASK_SWITCH -void vtime_task_switch(struct task_struct *prev) +void vtime_common_task_switch(struct task_struct *prev) { - if (!vtime_accounting_enabled()) - return; - if (is_idle_task(prev)) vtime_account_idle(prev); else @@ -417,11 +414,8 @@ void vtime_task_switch(struct task_struct *prev) * vtime_account(). */ #ifndef __ARCH_HAS_VTIME_ACCOUNT -void vtime_account_irq_enter(struct task_struct *tsk) +void vtime_common_account_irq_enter(struct task_struct *tsk) { - if (!vtime_accounting_enabled()) - return; - if (!in_interrupt()) { /* * If we interrupted user, context_tracking_in_user() @@ -441,7 +435,7 @@ void vtime_account_irq_enter(struct task_struct *tsk) } vtime_account_system(tsk); } -EXPORT_SYMBOL_GPL(vtime_account_irq_enter); +EXPORT_SYMBOL_GPL(vtime_common_account_irq_enter); #endif /* __ARCH_HAS_VTIME_ACCOUNT */ #endif /* CONFIG_VIRT_CPU_ACCOUNTING */ @@ -528,9 +522,8 @@ static cputime_t scale_stime(u64 stime, u64 rtime, u64 total) for (;;) { /* Make sure "rtime" is the bigger of stime/rtime */ - if (stime > rtime) { - u64 tmp = rtime; rtime = stime; stime = tmp; - } + if (stime > rtime) + swap(rtime, stime); /* Make sure 'total' fits in 32 bits */ if (total >> 32) @@ -637,9 +630,6 @@ void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st) cputime_adjust(&cputime, &p->prev_cputime, ut, st); } -/* - * Must be called with siglock held. - */ void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st) { struct task_cputime cputime; @@ -681,23 +671,17 @@ static void __vtime_account_system(struct task_struct *tsk) void vtime_account_system(struct task_struct *tsk) { - if (!vtime_accounting_enabled()) - return; - write_seqlock(&tsk->vtime_seqlock); __vtime_account_system(tsk); write_sequnlock(&tsk->vtime_seqlock); } -void vtime_account_irq_exit(struct task_struct *tsk) +void vtime_gen_account_irq_exit(struct task_struct *tsk) { - if (!vtime_accounting_enabled()) - return; - write_seqlock(&tsk->vtime_seqlock); + __vtime_account_system(tsk); if (context_tracking_in_user()) tsk->vtime_snap_whence = VTIME_USER; - __vtime_account_system(tsk); write_sequnlock(&tsk->vtime_seqlock); } @@ -705,12 +689,8 @@ void vtime_account_user(struct task_struct *tsk) { cputime_t delta_cpu; - if (!vtime_accounting_enabled()) - return; - - delta_cpu = get_vtime_delta(tsk); - write_seqlock(&tsk->vtime_seqlock); + delta_cpu = get_vtime_delta(tsk); tsk->vtime_snap_whence = VTIME_SYS; account_user_time(tsk, delta_cpu, cputime_to_scaled(delta_cpu)); write_sequnlock(&tsk->vtime_seqlock); @@ -718,22 +698,27 @@ void vtime_account_user(struct task_struct *tsk) void vtime_user_enter(struct task_struct *tsk) { - if (!vtime_accounting_enabled()) - return; - write_seqlock(&tsk->vtime_seqlock); - tsk->vtime_snap_whence = VTIME_USER; __vtime_account_system(tsk); + tsk->vtime_snap_whence = VTIME_USER; write_sequnlock(&tsk->vtime_seqlock); } void vtime_guest_enter(struct task_struct *tsk) { + /* + * The flags must be updated under the lock with + * the vtime_snap flush and update. + * That enforces a right ordering and update sequence + * synchronization against the reader (task_gtime()) + * that can thus safely catch up with a tickless delta. + */ write_seqlock(&tsk->vtime_seqlock); __vtime_account_system(tsk); current->flags |= PF_VCPU; write_sequnlock(&tsk->vtime_seqlock); } +EXPORT_SYMBOL_GPL(vtime_guest_enter); void vtime_guest_exit(struct task_struct *tsk) { @@ -742,6 +727,7 @@ void vtime_guest_exit(struct task_struct *tsk) current->flags &= ~PF_VCPU; write_sequnlock(&tsk->vtime_seqlock); } +EXPORT_SYMBOL_GPL(vtime_guest_exit); void vtime_account_idle(struct task_struct *tsk) { @@ -750,11 +736,6 @@ void vtime_account_idle(struct task_struct *tsk) account_idle_time(delta_cpu); } -bool vtime_accounting_enabled(void) -{ - return context_tracking_active(); -} - void arch_vtime_task_switch(struct task_struct *prev) { write_seqlock(&prev->vtime_seqlock); diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c new file mode 100644 index 0000000000000..4ea8dfcce1299 --- /dev/null +++ b/kernel/sched/deadline.c @@ -0,0 +1,1800 @@ +/* + * Deadline Scheduling Class (SCHED_DEADLINE) + * + * Earliest Deadline First (EDF) + Constant Bandwidth Server (CBS). + * + * Tasks that periodically executes their instances for less than their + * runtime won't miss any of their deadlines. + * Tasks that are not periodic or sporadic or that tries to execute more + * than their reserved bandwidth will be slowed down (and may potentially + * miss some of their deadlines), and won't affect any other task. + * + * Copyright (C) 2012 Dario Faggioli , + * Juri Lelli , + * Michael Trimarchi , + * Fabio Checconi + */ +#include "sched.h" + +#include + +struct dl_bandwidth def_dl_bandwidth; + +static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se) +{ + return container_of(dl_se, struct task_struct, dl); +} + +static inline struct rq *rq_of_dl_rq(struct dl_rq *dl_rq) +{ + return container_of(dl_rq, struct rq, dl); +} + +static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se) +{ + struct task_struct *p = dl_task_of(dl_se); + struct rq *rq = task_rq(p); + + return &rq->dl; +} + +static inline int on_dl_rq(struct sched_dl_entity *dl_se) +{ + return !RB_EMPTY_NODE(&dl_se->rb_node); +} + +static inline int is_leftmost(struct task_struct *p, struct dl_rq *dl_rq) +{ + struct sched_dl_entity *dl_se = &p->dl; + + return dl_rq->rb_leftmost == &dl_se->rb_node; +} + +void init_dl_bandwidth(struct dl_bandwidth *dl_b, u64 period, u64 runtime) +{ + raw_spin_lock_init(&dl_b->dl_runtime_lock); + dl_b->dl_period = period; + dl_b->dl_runtime = runtime; +} + +void init_dl_bw(struct dl_bw *dl_b) +{ + raw_spin_lock_init(&dl_b->lock); + raw_spin_lock(&def_dl_bandwidth.dl_runtime_lock); + if (global_rt_runtime() == RUNTIME_INF) + dl_b->bw = -1; + else + dl_b->bw = to_ratio(global_rt_period(), global_rt_runtime()); + raw_spin_unlock(&def_dl_bandwidth.dl_runtime_lock); + dl_b->total_bw = 0; +} + +void init_dl_rq(struct dl_rq *dl_rq, struct rq *rq) +{ + dl_rq->rb_root = RB_ROOT; + +#ifdef CONFIG_SMP + /* zero means no -deadline tasks */ + dl_rq->earliest_dl.curr = dl_rq->earliest_dl.next = 0; + + dl_rq->dl_nr_migratory = 0; + dl_rq->overloaded = 0; + dl_rq->pushable_dl_tasks_root = RB_ROOT; +#else + init_dl_bw(&dl_rq->dl_bw); +#endif +} + +#ifdef CONFIG_SMP + +static inline int dl_overloaded(struct rq *rq) +{ + return atomic_read(&rq->rd->dlo_count); +} + +static inline void dl_set_overload(struct rq *rq) +{ + if (!rq->online) + return; + + cpumask_set_cpu(rq->cpu, rq->rd->dlo_mask); + /* + * Must be visible before the overload count is + * set (as in sched_rt.c). + * + * Matched by the barrier in pull_dl_task(). + */ + smp_wmb(); + atomic_inc(&rq->rd->dlo_count); +} + +static inline void dl_clear_overload(struct rq *rq) +{ + if (!rq->online) + return; + + atomic_dec(&rq->rd->dlo_count); + cpumask_clear_cpu(rq->cpu, rq->rd->dlo_mask); +} + +static void update_dl_migration(struct dl_rq *dl_rq) +{ + if (dl_rq->dl_nr_migratory && dl_rq->dl_nr_running > 1) { + if (!dl_rq->overloaded) { + dl_set_overload(rq_of_dl_rq(dl_rq)); + dl_rq->overloaded = 1; + } + } else if (dl_rq->overloaded) { + dl_clear_overload(rq_of_dl_rq(dl_rq)); + dl_rq->overloaded = 0; + } +} + +static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) +{ + struct task_struct *p = dl_task_of(dl_se); + + if (p->nr_cpus_allowed > 1) + dl_rq->dl_nr_migratory++; + + update_dl_migration(dl_rq); +} + +static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) +{ + struct task_struct *p = dl_task_of(dl_se); + + if (p->nr_cpus_allowed > 1) + dl_rq->dl_nr_migratory--; + + update_dl_migration(dl_rq); +} + +/* + * The list of pushable -deadline task is not a plist, like in + * sched_rt.c, it is an rb-tree with tasks ordered by deadline. + */ +static void enqueue_pushable_dl_task(struct rq *rq, struct task_struct *p) +{ + struct dl_rq *dl_rq = &rq->dl; + struct rb_node **link = &dl_rq->pushable_dl_tasks_root.rb_node; + struct rb_node *parent = NULL; + struct task_struct *entry; + int leftmost = 1; + + BUG_ON(!RB_EMPTY_NODE(&p->pushable_dl_tasks)); + + while (*link) { + parent = *link; + entry = rb_entry(parent, struct task_struct, + pushable_dl_tasks); + if (dl_entity_preempt(&p->dl, &entry->dl)) + link = &parent->rb_left; + else { + link = &parent->rb_right; + leftmost = 0; + } + } + + if (leftmost) + dl_rq->pushable_dl_tasks_leftmost = &p->pushable_dl_tasks; + + rb_link_node(&p->pushable_dl_tasks, parent, link); + rb_insert_color(&p->pushable_dl_tasks, &dl_rq->pushable_dl_tasks_root); +} + +static void dequeue_pushable_dl_task(struct rq *rq, struct task_struct *p) +{ + struct dl_rq *dl_rq = &rq->dl; + + if (RB_EMPTY_NODE(&p->pushable_dl_tasks)) + return; + + if (dl_rq->pushable_dl_tasks_leftmost == &p->pushable_dl_tasks) { + struct rb_node *next_node; + + next_node = rb_next(&p->pushable_dl_tasks); + dl_rq->pushable_dl_tasks_leftmost = next_node; + } + + rb_erase(&p->pushable_dl_tasks, &dl_rq->pushable_dl_tasks_root); + RB_CLEAR_NODE(&p->pushable_dl_tasks); +} + +static inline int has_pushable_dl_tasks(struct rq *rq) +{ + return !RB_EMPTY_ROOT(&rq->dl.pushable_dl_tasks_root); +} + +static int push_dl_task(struct rq *rq); + +static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev) +{ + return dl_task(prev); +} + +static inline void set_post_schedule(struct rq *rq) +{ + rq->post_schedule = has_pushable_dl_tasks(rq); +} + +static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq); + +static struct rq *dl_task_offline_migration(struct rq *rq, struct task_struct *p) +{ + struct rq *later_rq = NULL; + bool fallback = false; + + later_rq = find_lock_later_rq(p, rq); + + if (!later_rq) { + int cpu; + + /* + * If we cannot preempt any rq, fall back to pick any + * online cpu. + */ + fallback = true; + cpu = cpumask_any_and(cpu_active_mask, tsk_cpus_allowed(p)); + if (cpu >= nr_cpu_ids) { + /* + * Fail to find any suitable cpu. + * The task will never come back! + */ + BUG_ON(dl_bandwidth_enabled()); + + /* + * If admission control is disabled we + * try a little harder to let the task + * run. + */ + cpu = cpumask_any(cpu_active_mask); + } + later_rq = cpu_rq(cpu); + double_lock_balance(rq, later_rq); + } + + /* + * By now the task is replenished and enqueued; migrate it. + */ + deactivate_task(rq, p, 0); + set_task_cpu(p, later_rq->cpu); + activate_task(later_rq, p, 0); + + if (!fallback) + resched_curr(later_rq); + + double_unlock_balance(later_rq, rq); + + return later_rq; +} + +#else + +static inline +void enqueue_pushable_dl_task(struct rq *rq, struct task_struct *p) +{ +} + +static inline +void dequeue_pushable_dl_task(struct rq *rq, struct task_struct *p) +{ +} + +static inline +void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) +{ +} + +static inline +void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) +{ +} + +static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev) +{ + return false; +} + +static inline int pull_dl_task(struct rq *rq) +{ + return 0; +} + +static inline void set_post_schedule(struct rq *rq) +{ +} +#endif /* CONFIG_SMP */ + +static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags); +static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags); +static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p, + int flags); + +/* + * We are being explicitly informed that a new instance is starting, + * and this means that: + * - the absolute deadline of the entity has to be placed at + * current time + relative deadline; + * - the runtime of the entity has to be set to the maximum value. + * + * The capability of specifying such event is useful whenever a -deadline + * entity wants to (try to!) synchronize its behaviour with the scheduler's + * one, and to (try to!) reconcile itself with its own scheduling + * parameters. + */ +static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se, + struct sched_dl_entity *pi_se) +{ + struct dl_rq *dl_rq = dl_rq_of_se(dl_se); + struct rq *rq = rq_of_dl_rq(dl_rq); + + WARN_ON(!dl_se->dl_new || dl_se->dl_throttled); + + /* + * We use the regular wall clock time to set deadlines in the + * future; in fact, we must consider execution overheads (time + * spent on hardirq context, etc.). + */ + dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; + dl_se->runtime = pi_se->dl_runtime; + dl_se->dl_new = 0; +} + +/* + * Pure Earliest Deadline First (EDF) scheduling does not deal with the + * possibility of a entity lasting more than what it declared, and thus + * exhausting its runtime. + * + * Here we are interested in making runtime overrun possible, but we do + * not want a entity which is misbehaving to affect the scheduling of all + * other entities. + * Therefore, a budgeting strategy called Constant Bandwidth Server (CBS) + * is used, in order to confine each entity within its own bandwidth. + * + * This function deals exactly with that, and ensures that when the runtime + * of a entity is replenished, its deadline is also postponed. That ensures + * the overrunning entity can't interfere with other entity in the system and + * can't make them miss their deadlines. Reasons why this kind of overruns + * could happen are, typically, a entity voluntarily trying to overcome its + * runtime, or it just underestimated it during sched_setattr(). + */ +static void replenish_dl_entity(struct sched_dl_entity *dl_se, + struct sched_dl_entity *pi_se) +{ + struct dl_rq *dl_rq = dl_rq_of_se(dl_se); + struct rq *rq = rq_of_dl_rq(dl_rq); + + BUG_ON(pi_se->dl_runtime <= 0); + + /* + * This could be the case for a !-dl task that is boosted. + * Just go with full inherited parameters. + */ + if (dl_se->dl_deadline == 0) { + dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; + dl_se->runtime = pi_se->dl_runtime; + } + + /* + * We keep moving the deadline away until we get some + * available runtime for the entity. This ensures correct + * handling of situations where the runtime overrun is + * arbitrary large. + */ + while (dl_se->runtime <= 0) { + dl_se->deadline += pi_se->dl_period; + dl_se->runtime += pi_se->dl_runtime; + } + + /* + * At this point, the deadline really should be "in + * the future" with respect to rq->clock. If it's + * not, we are, for some reason, lagging too much! + * Anyway, after having warn userspace abut that, + * we still try to keep the things running by + * resetting the deadline and the budget of the + * entity. + */ + if (dl_time_before(dl_se->deadline, rq_clock(rq))) { + //TJK printk_deferred_once("sched: DL replenish lagged to much\n"); + dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; + dl_se->runtime = pi_se->dl_runtime; + } + + if (dl_se->dl_yielded) + dl_se->dl_yielded = 0; + if (dl_se->dl_throttled) + dl_se->dl_throttled = 0; +} + +/* + * Here we check if --at time t-- an entity (which is probably being + * [re]activated or, in general, enqueued) can use its remaining runtime + * and its current deadline _without_ exceeding the bandwidth it is + * assigned (function returns true if it can't). We are in fact applying + * one of the CBS rules: when a task wakes up, if the residual runtime + * over residual deadline fits within the allocated bandwidth, then we + * can keep the current (absolute) deadline and residual budget without + * disrupting the schedulability of the system. Otherwise, we should + * refill the runtime and set the deadline a period in the future, + * because keeping the current (absolute) deadline of the task would + * result in breaking guarantees promised to other tasks (refer to + * Documentation/scheduler/sched-deadline.txt for more informations). + * + * This function returns true if: + * + * runtime / (deadline - t) > dl_runtime / dl_period , + * + * IOW we can't recycle current parameters. + * + * Notice that the bandwidth check is done against the period. For + * task with deadline equal to period this is the same of using + * dl_deadline instead of dl_period in the equation above. + */ +static bool dl_entity_overflow(struct sched_dl_entity *dl_se, + struct sched_dl_entity *pi_se, u64 t) +{ + u64 left, right; + + /* + * left and right are the two sides of the equation above, + * after a bit of shuffling to use multiplications instead + * of divisions. + * + * Note that none of the time values involved in the two + * multiplications are absolute: dl_deadline and dl_runtime + * are the relative deadline and the maximum runtime of each + * instance, runtime is the runtime left for the last instance + * and (deadline - t), since t is rq->clock, is the time left + * to the (absolute) deadline. Even if overflowing the u64 type + * is very unlikely to occur in both cases, here we scale down + * as we want to avoid that risk at all. Scaling down by 10 + * means that we reduce granularity to 1us. We are fine with it, + * since this is only a true/false check and, anyway, thinking + * of anything below microseconds resolution is actually fiction + * (but still we want to give the user that illusion >;). + */ + left = (pi_se->dl_period >> DL_SCALE) * (dl_se->runtime >> DL_SCALE); + right = ((dl_se->deadline - t) >> DL_SCALE) * + (pi_se->dl_runtime >> DL_SCALE); + + return dl_time_before(right, left); +} + +/* + * When a -deadline entity is queued back on the runqueue, its runtime and + * deadline might need updating. + * + * The policy here is that we update the deadline of the entity only if: + * - the current deadline is in the past, + * - using the remaining runtime with the current deadline would make + * the entity exceed its bandwidth. + */ +static void update_dl_entity(struct sched_dl_entity *dl_se, + struct sched_dl_entity *pi_se) +{ + struct dl_rq *dl_rq = dl_rq_of_se(dl_se); + struct rq *rq = rq_of_dl_rq(dl_rq); + + /* + * The arrival of a new instance needs special treatment, i.e., + * the actual scheduling parameters have to be "renewed". + */ + if (dl_se->dl_new) { + setup_new_dl_entity(dl_se, pi_se); + return; + } + + if (dl_time_before(dl_se->deadline, rq_clock(rq)) || + dl_entity_overflow(dl_se, pi_se, rq_clock(rq))) { + dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; + dl_se->runtime = pi_se->dl_runtime; + } +} + +/* + * If the entity depleted all its runtime, and if we want it to sleep + * while waiting for some new execution time to become available, we + * set the bandwidth enforcement timer to the replenishment instant + * and try to activate it. + * + * Notice that it is important for the caller to know if the timer + * actually started or not (i.e., the replenishment instant is in + * the future or in the past). + */ +static int start_dl_timer(struct task_struct *p) +{ + struct sched_dl_entity *dl_se = &p->dl; + struct hrtimer *timer = &dl_se->dl_timer; + struct rq *rq = task_rq(p); + ktime_t now, act; + s64 delta; + + lockdep_assert_held(&rq->lock); + + /* + * We want the timer to fire at the deadline, but considering + * that it is actually coming from rq->clock and not from + * hrtimer's time base reading. + */ + act = ns_to_ktime(dl_se->deadline); + now = hrtimer_cb_get_time(timer); + delta = ktime_to_ns(now) - rq_clock(rq); + act = ktime_add_ns(act, delta); + + /* + * If the expiry time already passed, e.g., because the value + * chosen as the deadline is too small, don't even try to + * start the timer in the past! + */ + if (ktime_us_delta(act, now) < 0) + return 0; + + /* + * !enqueued will guarantee another callback; even if one is already in + * progress. This ensures a balanced {get,put}_task_struct(). + * + * The race against __run_timer() clearing the enqueued state is + * harmless because we're holding task_rq()->lock, therefore the timer + * expiring after we've done the check will wait on its task_rq_lock() + * and observe our state. + */ + if (!hrtimer_is_queued(timer)) { + get_task_struct(p); + hrtimer_start(timer, act, HRTIMER_MODE_ABS); + } + + return 1; +} + +/* + * This is the bandwidth enforcement timer callback. If here, we know + * a task is not on its dl_rq, since the fact that the timer was running + * means the task is throttled and needs a runtime replenishment. + * + * However, what we actually do depends on the fact the task is active, + * (it is on its rq) or has been removed from there by a call to + * dequeue_task_dl(). In the former case we must issue the runtime + * replenishment and add the task back to the dl_rq; in the latter, we just + * do nothing but clearing dl_throttled, so that runtime and deadline + * updating (and the queueing back to dl_rq) will be done by the + * next call to enqueue_task_dl(). + */ +static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) +{ + struct sched_dl_entity *dl_se = container_of(timer, + struct sched_dl_entity, + dl_timer); + struct task_struct *p = dl_task_of(dl_se); + struct rq *rq; +again: + rq = task_rq(p); + raw_spin_lock(&rq->lock); + + if (rq != task_rq(p)) { + /* Task was moved, retrying. */ + raw_spin_unlock(&rq->lock); + goto again; + } + + /* + * The task might have changed its scheduling policy to something + * different than SCHED_DEADLINE (through switched_fromd_dl()). + */ + if (!dl_task(p)) { + __dl_clear_params(p); + goto unlock; + } + + /* + * This is possible if switched_from_dl() raced against a running + * callback that took the above !dl_task() path and we've since then + * switched back into SCHED_DEADLINE. + * + * There's nothing to do except drop our task reference. + */ + if (dl_se->dl_new) + goto unlock; + + /* + * The task might have been boosted by someone else and might be in the + * boosting/deboosting path, its not throttled. + */ + if (dl_se->dl_boosted) + goto unlock; + + /* + * Spurious timer due to start_dl_timer() race; or we already received + * a replenishment from rt_mutex_setprio(). + */ + if (!dl_se->dl_throttled) + goto unlock; + + sched_clock_tick(); + update_rq_clock(rq); + + /* + * If the throttle happened during sched-out; like: + * + * schedule() + * deactivate_task() + * dequeue_task_dl() + * update_curr_dl() + * start_dl_timer() + * __dequeue_task_dl() + * prev->on_rq = 0; + * + * We can be both throttled and !queued. Replenish the counter + * but do not enqueue -- wait for our wakeup to do that. + */ + if (!task_on_rq_queued(p)) { + replenish_dl_entity(dl_se, dl_se); + goto unlock; + } + + enqueue_task_dl(rq, p, ENQUEUE_REPLENISH); + if (dl_task(rq->curr)) + check_preempt_curr_dl(rq, p, 0); + else + resched_curr(rq); + +#ifdef CONFIG_SMP + /* + * Perform balancing operations here; after the replenishments. We + * cannot drop rq->lock before this, otherwise the assertion in + * start_dl_timer() about not missing updates is not true. + * + * If we find that the rq the task was on is no longer available, we + * need to select a new rq. + * + * XXX figure out if select_task_rq_dl() deals with offline cpus. + */ + if (unlikely(!rq->online)) + rq = dl_task_offline_migration(rq, p); + + /* + * Queueing this task back might have overloaded rq, check if we need + * to kick someone away. + */ + if (has_pushable_dl_tasks(rq)) + push_dl_task(rq); +#endif + +unlock: + raw_spin_unlock(&rq->lock); + + /* + * This can free the task_struct, including this hrtimer, do not touch + * anything related to that after this. + */ + put_task_struct(p); + + return HRTIMER_NORESTART; +} + +void init_dl_task_timer(struct sched_dl_entity *dl_se) +{ + struct hrtimer *timer = &dl_se->dl_timer; + + hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + timer->function = dl_task_timer; +} + +static +int dl_runtime_exceeded(struct rq *rq, struct sched_dl_entity *dl_se) +{ + return (dl_se->runtime <= 0); +} + +extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq); + +/* + * Update the current task's runtime statistics (provided it is still + * a -deadline task and has not been removed from the dl_rq). + */ +static void update_curr_dl(struct rq *rq) +{ + struct task_struct *curr = rq->curr; + struct sched_dl_entity *dl_se = &curr->dl; + u64 delta_exec; + + if (!dl_task(curr) || !on_dl_rq(dl_se)) + return; + + /* + * Consumed budget is computed considering the time as + * observed by schedulable tasks (excluding time spent + * in hardirq context, etc.). Deadlines are instead + * computed using hard walltime. This seems to be the more + * natural solution, but the full ramifications of this + * approach need further study. + */ + delta_exec = rq_clock_task(rq) - curr->se.exec_start; + if (unlikely((s64)delta_exec <= 0)) + return; + + schedstat_set(curr->se.statistics.exec_max, + max(curr->se.statistics.exec_max, delta_exec)); + + curr->se.sum_exec_runtime += delta_exec; + account_group_exec_runtime(curr, delta_exec); + + curr->se.exec_start = rq_clock_task(rq); + cpuacct_charge(curr, delta_exec); + + sched_rt_avg_update(rq, delta_exec); + + dl_se->runtime -= dl_se->dl_yielded ? 0 : delta_exec; + if (dl_runtime_exceeded(rq, dl_se)) { + dl_se->dl_throttled = 1; + __dequeue_task_dl(rq, curr, 0); + if (unlikely(dl_se->dl_boosted || !start_dl_timer(curr))) + enqueue_task_dl(rq, curr, ENQUEUE_REPLENISH); + + if (!is_leftmost(curr, &rq->dl)) + resched_curr(rq); + } + + /* + * Because -- for now -- we share the rt bandwidth, we need to + * account our runtime there too, otherwise actual rt tasks + * would be able to exceed the shared quota. + * + * Account to the root rt group for now. + * + * The solution we're working towards is having the RT groups scheduled + * using deadline servers -- however there's a few nasties to figure + * out before that can happen. + */ + if (rt_bandwidth_enabled()) { + struct rt_rq *rt_rq = &rq->rt; + + raw_spin_lock(&rt_rq->rt_runtime_lock); + /* + * We'll let actual RT tasks worry about the overflow here, we + * have our own CBS to keep us inline; only account when RT + * bandwidth is relevant. + */ + if (sched_rt_bandwidth_account(rt_rq)) + rt_rq->rt_time += delta_exec; + raw_spin_unlock(&rt_rq->rt_runtime_lock); + } +} + +#ifdef CONFIG_SMP + +static struct task_struct *pick_next_earliest_dl_task(struct rq *rq, int cpu); + +static inline u64 next_deadline(struct rq *rq) +{ + struct task_struct *next = pick_next_earliest_dl_task(rq, rq->cpu); + + if (next && dl_prio(next->prio)) + return next->dl.deadline; + else + return 0; +} + +static void inc_dl_deadline(struct dl_rq *dl_rq, u64 deadline) +{ + struct rq *rq = rq_of_dl_rq(dl_rq); + + if (dl_rq->earliest_dl.curr == 0 || + dl_time_before(deadline, dl_rq->earliest_dl.curr)) { + /* + * If the dl_rq had no -deadline tasks, or if the new task + * has shorter deadline than the current one on dl_rq, we + * know that the previous earliest becomes our next earliest, + * as the new task becomes the earliest itself. + */ + dl_rq->earliest_dl.next = dl_rq->earliest_dl.curr; + dl_rq->earliest_dl.curr = deadline; + cpudl_set(&rq->rd->cpudl, rq->cpu, deadline, 1); + } else if (dl_rq->earliest_dl.next == 0 || + dl_time_before(deadline, dl_rq->earliest_dl.next)) { + /* + * On the other hand, if the new -deadline task has a + * a later deadline than the earliest one on dl_rq, but + * it is earlier than the next (if any), we must + * recompute the next-earliest. + */ + dl_rq->earliest_dl.next = next_deadline(rq); + } +} + +static void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline) +{ + struct rq *rq = rq_of_dl_rq(dl_rq); + + /* + * Since we may have removed our earliest (and/or next earliest) + * task we must recompute them. + */ + if (!dl_rq->dl_nr_running) { + dl_rq->earliest_dl.curr = 0; + dl_rq->earliest_dl.next = 0; + cpudl_set(&rq->rd->cpudl, rq->cpu, 0, 0); + } else { + struct rb_node *leftmost = dl_rq->rb_leftmost; + struct sched_dl_entity *entry; + + entry = rb_entry(leftmost, struct sched_dl_entity, rb_node); + dl_rq->earliest_dl.curr = entry->deadline; + dl_rq->earliest_dl.next = next_deadline(rq); + cpudl_set(&rq->rd->cpudl, rq->cpu, entry->deadline, 1); + } +} + +#else + +static inline void inc_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {} +static inline void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {} + +#endif /* CONFIG_SMP */ + +static inline +void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) +{ + int prio = dl_task_of(dl_se)->prio; + u64 deadline = dl_se->deadline; + + WARN_ON(!dl_prio(prio)); + dl_rq->dl_nr_running++; + add_nr_running(rq_of_dl_rq(dl_rq), 1); + + inc_dl_deadline(dl_rq, deadline); + inc_dl_migration(dl_se, dl_rq); +} + +static inline +void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) +{ + int prio = dl_task_of(dl_se)->prio; + + WARN_ON(!dl_prio(prio)); + WARN_ON(!dl_rq->dl_nr_running); + dl_rq->dl_nr_running--; + sub_nr_running(rq_of_dl_rq(dl_rq), 1); + + dec_dl_deadline(dl_rq, dl_se->deadline); + dec_dl_migration(dl_se, dl_rq); +} + +static void __enqueue_dl_entity(struct sched_dl_entity *dl_se) +{ + struct dl_rq *dl_rq = dl_rq_of_se(dl_se); + struct rb_node **link = &dl_rq->rb_root.rb_node; + struct rb_node *parent = NULL; + struct sched_dl_entity *entry; + int leftmost = 1; + + BUG_ON(!RB_EMPTY_NODE(&dl_se->rb_node)); + + while (*link) { + parent = *link; + entry = rb_entry(parent, struct sched_dl_entity, rb_node); + if (dl_time_before(dl_se->deadline, entry->deadline)) + link = &parent->rb_left; + else { + link = &parent->rb_right; + leftmost = 0; + } + } + + if (leftmost) + dl_rq->rb_leftmost = &dl_se->rb_node; + + rb_link_node(&dl_se->rb_node, parent, link); + rb_insert_color(&dl_se->rb_node, &dl_rq->rb_root); + + inc_dl_tasks(dl_se, dl_rq); +} + +static void __dequeue_dl_entity(struct sched_dl_entity *dl_se) +{ + struct dl_rq *dl_rq = dl_rq_of_se(dl_se); + + if (RB_EMPTY_NODE(&dl_se->rb_node)) + return; + + if (dl_rq->rb_leftmost == &dl_se->rb_node) { + struct rb_node *next_node; + + next_node = rb_next(&dl_se->rb_node); + dl_rq->rb_leftmost = next_node; + } + + rb_erase(&dl_se->rb_node, &dl_rq->rb_root); + RB_CLEAR_NODE(&dl_se->rb_node); + + dec_dl_tasks(dl_se, dl_rq); +} + +static void +enqueue_dl_entity(struct sched_dl_entity *dl_se, + struct sched_dl_entity *pi_se, int flags) +{ + BUG_ON(on_dl_rq(dl_se)); + + /* + * If this is a wakeup or a new instance, the scheduling + * parameters of the task might need updating. Otherwise, + * we want a replenishment of its runtime. + */ + if (dl_se->dl_new || flags & ENQUEUE_WAKEUP) + update_dl_entity(dl_se, pi_se); + else if (flags & ENQUEUE_REPLENISH) + replenish_dl_entity(dl_se, pi_se); + + __enqueue_dl_entity(dl_se); +} + +static void dequeue_dl_entity(struct sched_dl_entity *dl_se) +{ + __dequeue_dl_entity(dl_se); +} + +static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) +{ + struct task_struct *pi_task = rt_mutex_get_top_task(p); + struct sched_dl_entity *pi_se = &p->dl; + + /* + * Use the scheduling parameters of the top pi-waiter + * task if we have one and its (relative) deadline is + * smaller than our one... OTW we keep our runtime and + * deadline. + */ + if (pi_task && p->dl.dl_boosted && dl_prio(pi_task->normal_prio)) { + pi_se = &pi_task->dl; + } else if (!dl_prio(p->normal_prio)) { + /* + * Special case in which we have a !SCHED_DEADLINE task + * that is going to be deboosted, but exceedes its + * runtime while doing so. No point in replenishing + * it, as it's going to return back to its original + * scheduling class after this. + */ + BUG_ON(!p->dl.dl_boosted || flags != ENQUEUE_REPLENISH); + return; + } + + /* + * If p is throttled, we do nothing. In fact, if it exhausted + * its budget it needs a replenishment and, since it now is on + * its rq, the bandwidth timer callback (which clearly has not + * run yet) will take care of this. + */ + if (p->dl.dl_throttled && !(flags & ENQUEUE_REPLENISH)) + return; + + enqueue_dl_entity(&p->dl, pi_se, flags); + + if (!task_current(rq, p) && p->nr_cpus_allowed > 1) + enqueue_pushable_dl_task(rq, p); +} + +static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags) +{ + dequeue_dl_entity(&p->dl); + dequeue_pushable_dl_task(rq, p); +} + +static void dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags) +{ + update_curr_dl(rq); + __dequeue_task_dl(rq, p, flags); +} + +/* + * Yield task semantic for -deadline tasks is: + * + * get off from the CPU until our next instance, with + * a new runtime. This is of little use now, since we + * don't have a bandwidth reclaiming mechanism. Anyway, + * bandwidth reclaiming is planned for the future, and + * yield_task_dl will indicate that some spare budget + * is available for other task instances to use it. + */ +static void yield_task_dl(struct rq *rq) +{ + struct task_struct *p = rq->curr; + + /* + * We make the task go to sleep until its current deadline by + * forcing its runtime to zero. This way, update_curr_dl() stops + * it and the bandwidth timer will wake it up and will give it + * new scheduling parameters (thanks to dl_yielded=1). + */ + if (p->dl.runtime > 0) { + rq->curr->dl.dl_yielded = 1; + p->dl.runtime = 0; + } + update_curr_dl(rq); +} + +#ifdef CONFIG_SMP + +static int find_later_rq(struct task_struct *task); + +static int +select_task_rq_dl(struct task_struct *p, int cpu, int sd_flag, int flags) +{ + struct task_struct *curr; + struct rq *rq; + + if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK) + goto out; + + rq = cpu_rq(cpu); + + rcu_read_lock(); + curr = ACCESS_ONCE(rq->curr); /* unlocked access */ + + /* + * If we are dealing with a -deadline task, we must + * decide where to wake it up. + * If it has a later deadline and the current task + * on this rq can't move (provided the waking task + * can!) we prefer to send it somewhere else. On the + * other hand, if it has a shorter deadline, we + * try to make it stay here, it might be important. + */ + if (unlikely(dl_task(curr)) && + (curr->nr_cpus_allowed < 2 || + !dl_entity_preempt(&p->dl, &curr->dl)) && + (p->nr_cpus_allowed > 1)) { + int target = find_later_rq(p); + + if (target != -1) + cpu = target; + } + rcu_read_unlock(); + +out: + return cpu; +} + +static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p) +{ + /* + * Current can't be migrated, useless to reschedule, + * let's hope p can move out. + */ + if (rq->curr->nr_cpus_allowed == 1 || + cpudl_find(&rq->rd->cpudl, rq->curr, NULL) == -1) + return; + + /* + * p is migratable, so let's not schedule it and + * see if it is pushed or pulled somewhere else. + */ + if (p->nr_cpus_allowed != 1 && + cpudl_find(&rq->rd->cpudl, p, NULL) != -1) + return; + + resched_curr(rq); +} + +static int pull_dl_task(struct rq *this_rq); + +#endif /* CONFIG_SMP */ + +/* + * Only called when both the current and waking task are -deadline + * tasks. + */ +static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p, + int flags) +{ + if (dl_entity_preempt(&p->dl, &rq->curr->dl)) { + resched_curr(rq); + return; + } + +#ifdef CONFIG_SMP + /* + * In the unlikely case current and p have the same deadline + * let us try to decide what's the best thing to do... + */ + if ((p->dl.deadline == rq->curr->dl.deadline) && + !test_tsk_need_resched(rq->curr)) + check_preempt_equal_dl(rq, p); +#endif /* CONFIG_SMP */ +} + +#ifdef CONFIG_SCHED_HRTICK +static void start_hrtick_dl(struct rq *rq, struct task_struct *p) +{ + hrtick_start(rq, p->dl.runtime); +} +#endif + +static struct sched_dl_entity *pick_next_dl_entity(struct rq *rq, + struct dl_rq *dl_rq) +{ + struct rb_node *left = dl_rq->rb_leftmost; + + if (!left) + return NULL; + + return rb_entry(left, struct sched_dl_entity, rb_node); +} + +struct task_struct *pick_next_task_dl(struct rq *rq, struct task_struct *prev) +{ + struct sched_dl_entity *dl_se; + struct task_struct *p; + struct dl_rq *dl_rq; + + dl_rq = &rq->dl; + + if (need_pull_dl_task(rq, prev)) { + pull_dl_task(rq); + /* + * pull_rt_task() can drop (and re-acquire) rq->lock; this + * means a stop task can slip in, in which case we need to + * re-start task selection. + */ + if (rq->stop && task_on_rq_queued(rq->stop)) + return RETRY_TASK; + } + + /* + * When prev is DL, we may throttle it in put_prev_task(). + * So, we update time before we check for dl_nr_running. + */ + if (prev->sched_class == &dl_sched_class) + update_curr_dl(rq); + + if (unlikely(!dl_rq->dl_nr_running)) + return NULL; + + put_prev_task(rq, prev); + + dl_se = pick_next_dl_entity(rq, dl_rq); + BUG_ON(!dl_se); + + p = dl_task_of(dl_se); + p->se.exec_start = rq_clock_task(rq); + + /* Running task will never be pushed. */ + dequeue_pushable_dl_task(rq, p); + +#ifdef CONFIG_SCHED_HRTICK + if (hrtick_enabled(rq)) + start_hrtick_dl(rq, p); +#endif + + set_post_schedule(rq); + + return p; +} + +static void put_prev_task_dl(struct rq *rq, struct task_struct *p) +{ + update_curr_dl(rq); + + if (on_dl_rq(&p->dl) && p->nr_cpus_allowed > 1) + enqueue_pushable_dl_task(rq, p); +} + +static void task_tick_dl(struct rq *rq, struct task_struct *p, int queued) +{ + update_curr_dl(rq); + +#ifdef CONFIG_SCHED_HRTICK + if (hrtick_enabled(rq) && queued && p->dl.runtime > 0) + start_hrtick_dl(rq, p); +#endif +} + +static void task_fork_dl(struct task_struct *p) +{ + /* + * SCHED_DEADLINE tasks cannot fork and this is achieved through + * sched_fork() + */ +} + +static void task_dead_dl(struct task_struct *p) +{ + struct dl_bw *dl_b = dl_bw_of(task_cpu(p)); + + /* + * Since we are TASK_DEAD we won't slip out of the domain! + */ + raw_spin_lock_irq(&dl_b->lock); + /* XXX we should retain the bw until 0-lag */ + dl_b->total_bw -= p->dl.dl_bw; + raw_spin_unlock_irq(&dl_b->lock); +} + +static void set_curr_task_dl(struct rq *rq) +{ + struct task_struct *p = rq->curr; + + p->se.exec_start = rq_clock_task(rq); + + /* You can't push away the running task */ + dequeue_pushable_dl_task(rq, p); +} + +#ifdef CONFIG_SMP + +/* Only try algorithms three times */ +#define DL_MAX_TRIES 3 + +static int pick_dl_task(struct rq *rq, struct task_struct *p, int cpu) +{ + if (!task_running(rq, p) && + cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) + return 1; + return 0; +} + +/* Returns the second earliest -deadline task, NULL otherwise */ +static struct task_struct *pick_next_earliest_dl_task(struct rq *rq, int cpu) +{ + struct rb_node *next_node = rq->dl.rb_leftmost; + struct sched_dl_entity *dl_se; + struct task_struct *p = NULL; + +next_node: + next_node = rb_next(next_node); + if (next_node) { + dl_se = rb_entry(next_node, struct sched_dl_entity, rb_node); + p = dl_task_of(dl_se); + + if (pick_dl_task(rq, p, cpu)) + return p; + + goto next_node; + } + + return NULL; +} + +static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask_dl); + +static int find_later_rq(struct task_struct *task) +{ + struct sched_domain *sd; + struct cpumask *later_mask = this_cpu_ptr(local_cpu_mask_dl); + int this_cpu = smp_processor_id(); + int best_cpu, cpu = task_cpu(task); + + /* Make sure the mask is initialized first */ + if (unlikely(!later_mask)) + return -1; + + if (task->nr_cpus_allowed == 1) + return -1; + + /* + * We have to consider system topology and task affinity + * first, then we can look for a suitable cpu. + */ + cpumask_copy(later_mask, task_rq(task)->rd->span); + cpumask_and(later_mask, later_mask, cpu_active_mask); + cpumask_and(later_mask, later_mask, &task->cpus_allowed); + best_cpu = cpudl_find(&task_rq(task)->rd->cpudl, + task, later_mask); + if (best_cpu == -1) + return -1; + + /* + * If we are here, some target has been found, + * the most suitable of which is cached in best_cpu. + * This is, among the runqueues where the current tasks + * have later deadlines than the task's one, the rq + * with the latest possible one. + * + * Now we check how well this matches with task's + * affinity and system topology. + * + * The last cpu where the task run is our first + * guess, since it is most likely cache-hot there. + */ + if (cpumask_test_cpu(cpu, later_mask)) + return cpu; + /* + * Check if this_cpu is to be skipped (i.e., it is + * not in the mask) or not. + */ + if (!cpumask_test_cpu(this_cpu, later_mask)) + this_cpu = -1; + + rcu_read_lock(); + for_each_domain(cpu, sd) { + if (sd->flags & SD_WAKE_AFFINE) { + + /* + * If possible, preempting this_cpu is + * cheaper than migrating. + */ + if (this_cpu != -1 && + cpumask_test_cpu(this_cpu, sched_domain_span(sd))) { + rcu_read_unlock(); + return this_cpu; + } + + /* + * Last chance: if best_cpu is valid and is + * in the mask, that becomes our choice. + */ + if (best_cpu < nr_cpu_ids && + cpumask_test_cpu(best_cpu, sched_domain_span(sd))) { + rcu_read_unlock(); + return best_cpu; + } + } + } + rcu_read_unlock(); + + /* + * At this point, all our guesses failed, we just return + * 'something', and let the caller sort the things out. + */ + if (this_cpu != -1) + return this_cpu; + + cpu = cpumask_any(later_mask); + if (cpu < nr_cpu_ids) + return cpu; + + return -1; +} + +/* Locks the rq it finds */ +static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq) +{ + struct rq *later_rq = NULL; + int tries; + int cpu; + + for (tries = 0; tries < DL_MAX_TRIES; tries++) { + cpu = find_later_rq(task); + + if ((cpu == -1) || (cpu == rq->cpu)) + break; + + later_rq = cpu_rq(cpu); + + /* Retry if something changed. */ + if (double_lock_balance(rq, later_rq)) { + if (unlikely(task_rq(task) != rq || + !cpumask_test_cpu(later_rq->cpu, + &task->cpus_allowed) || + task_running(rq, task) || + !task_on_rq_queued(task))) { + double_unlock_balance(rq, later_rq); + later_rq = NULL; + break; + } + } + + /* + * If the rq we found has no -deadline task, or + * its earliest one has a later deadline than our + * task, the rq is a good one. + */ + if (!later_rq->dl.dl_nr_running || + dl_time_before(task->dl.deadline, + later_rq->dl.earliest_dl.curr)) + break; + + /* Otherwise we try again. */ + double_unlock_balance(rq, later_rq); + later_rq = NULL; + } + + return later_rq; +} + +static struct task_struct *pick_next_pushable_dl_task(struct rq *rq) +{ + struct task_struct *p; + + if (!has_pushable_dl_tasks(rq)) + return NULL; + + p = rb_entry(rq->dl.pushable_dl_tasks_leftmost, + struct task_struct, pushable_dl_tasks); + + BUG_ON(rq->cpu != task_cpu(p)); + BUG_ON(task_current(rq, p)); + BUG_ON(p->nr_cpus_allowed <= 1); + + BUG_ON(!task_on_rq_queued(p)); + BUG_ON(!dl_task(p)); + + return p; +} + +/* + * See if the non running -deadline tasks on this rq + * can be sent to some other CPU where they can preempt + * and start executing. + */ +static int push_dl_task(struct rq *rq) +{ + struct task_struct *next_task; + struct rq *later_rq; + + if (!rq->dl.overloaded) + return 0; + + next_task = pick_next_pushable_dl_task(rq); + if (!next_task) + return 0; + +retry: + if (unlikely(next_task == rq->curr)) { + WARN_ON(1); + return 0; + } + + /* + * If next_task preempts rq->curr, and rq->curr + * can move away, it makes sense to just reschedule + * without going further in pushing next_task. + */ + if (dl_task(rq->curr) && + dl_time_before(next_task->dl.deadline, rq->curr->dl.deadline) && + rq->curr->nr_cpus_allowed > 1) { + resched_curr(rq); + return 0; + } + + /* We might release rq lock */ + get_task_struct(next_task); + + /* Will lock the rq it'll find */ + later_rq = find_lock_later_rq(next_task, rq); + if (!later_rq) { + struct task_struct *task; + + /* + * We must check all this again, since + * find_lock_later_rq releases rq->lock and it is + * then possible that next_task has migrated. + */ + task = pick_next_pushable_dl_task(rq); + if (task_cpu(next_task) == rq->cpu && task == next_task) { + /* + * The task is still there. We don't try + * again, some other cpu will pull it when ready. + */ + dequeue_pushable_dl_task(rq, next_task); + goto out; + } + + if (!task) + /* No more tasks */ + goto out; + + put_task_struct(next_task); + next_task = task; + goto retry; + } + + deactivate_task(rq, next_task, 0); + set_task_cpu(next_task, later_rq->cpu); + activate_task(later_rq, next_task, 0); + + resched_curr(later_rq); + + double_unlock_balance(rq, later_rq); + +out: + put_task_struct(next_task); + + return 1; +} + +static void push_dl_tasks(struct rq *rq) +{ + /* Terminates as it moves a -deadline task */ + while (push_dl_task(rq)) + ; +} + +static int pull_dl_task(struct rq *this_rq) +{ + int this_cpu = this_rq->cpu, ret = 0, cpu; + struct task_struct *p; + struct rq *src_rq; + u64 dmin = LONG_MAX; + + if (likely(!dl_overloaded(this_rq))) + return 0; + + /* + * Match the barrier from dl_set_overloaded; this guarantees that if we + * see overloaded we must also see the dlo_mask bit. + */ + smp_rmb(); + + for_each_cpu(cpu, this_rq->rd->dlo_mask) { + if (this_cpu == cpu) + continue; + + src_rq = cpu_rq(cpu); + + /* + * It looks racy, abd it is! However, as in sched_rt.c, + * we are fine with this. + */ + if (this_rq->dl.dl_nr_running && + dl_time_before(this_rq->dl.earliest_dl.curr, + src_rq->dl.earliest_dl.next)) + continue; + + /* Might drop this_rq->lock */ + double_lock_balance(this_rq, src_rq); + + /* + * If there are no more pullable tasks on the + * rq, we're done with it. + */ + if (src_rq->dl.dl_nr_running <= 1) + goto skip; + + p = pick_next_earliest_dl_task(src_rq, this_cpu); + + /* + * We found a task to be pulled if: + * - it preempts our current (if there's one), + * - it will preempt the last one we pulled (if any). + */ + if (p && dl_time_before(p->dl.deadline, dmin) && + (!this_rq->dl.dl_nr_running || + dl_time_before(p->dl.deadline, + this_rq->dl.earliest_dl.curr))) { + WARN_ON(p == src_rq->curr); + WARN_ON(!task_on_rq_queued(p)); + + /* + * Then we pull iff p has actually an earlier + * deadline than the current task of its runqueue. + */ + if (dl_time_before(p->dl.deadline, + src_rq->curr->dl.deadline)) + goto skip; + + ret = 1; + + deactivate_task(src_rq, p, 0); + set_task_cpu(p, this_cpu); + activate_task(this_rq, p, 0); + dmin = p->dl.deadline; + + /* Is there any other task even earlier? */ + } +skip: + double_unlock_balance(this_rq, src_rq); + } + + return ret; +} + +static void post_schedule_dl(struct rq *rq) +{ + push_dl_tasks(rq); +} + +/* + * Since the task is not running and a reschedule is not going to happen + * anytime soon on its runqueue, we try pushing it away now. + */ +static void task_woken_dl(struct rq *rq, struct task_struct *p) +{ + if (!task_running(rq, p) && + !test_tsk_need_resched(rq->curr) && + has_pushable_dl_tasks(rq) && + p->nr_cpus_allowed > 1 && + dl_task(rq->curr) && + (rq->curr->nr_cpus_allowed < 2 || + dl_entity_preempt(&rq->curr->dl, &p->dl))) { + push_dl_tasks(rq); + } +} + +static void set_cpus_allowed_dl(struct task_struct *p, + const struct cpumask *new_mask) +{ + struct rq *rq; + int weight; + + BUG_ON(!dl_task(p)); + + /* + * Update only if the task is actually running (i.e., + * it is on the rq AND it is not throttled). + */ + if (!on_dl_rq(&p->dl)) + return; + + weight = cpumask_weight(new_mask); + + /* + * Only update if the process changes its state from whether it + * can migrate or not. + */ + if ((p->nr_cpus_allowed > 1) == (weight > 1)) + return; + + rq = task_rq(p); + + /* + * The process used to be able to migrate OR it can now migrate + */ + if (weight <= 1) { + if (!task_current(rq, p)) + dequeue_pushable_dl_task(rq, p); + BUG_ON(!rq->dl.dl_nr_migratory); + rq->dl.dl_nr_migratory--; + } else { + if (!task_current(rq, p)) + enqueue_pushable_dl_task(rq, p); + rq->dl.dl_nr_migratory++; + } + + update_dl_migration(&rq->dl); +} + +/* Assumes rq->lock is held */ +static void rq_online_dl(struct rq *rq) +{ + if (rq->dl.overloaded) + dl_set_overload(rq); + + if (rq->dl.dl_nr_running > 0) + cpudl_set(&rq->rd->cpudl, rq->cpu, rq->dl.earliest_dl.curr, 1); +} + +/* Assumes rq->lock is held */ +static void rq_offline_dl(struct rq *rq) +{ + if (rq->dl.overloaded) + dl_clear_overload(rq); + + cpudl_set(&rq->rd->cpudl, rq->cpu, 0, 0); +} + +void init_sched_dl_class(void) +{ + unsigned int i; + + for_each_possible_cpu(i) + zalloc_cpumask_var_node(&per_cpu(local_cpu_mask_dl, i), + GFP_KERNEL, cpu_to_node(i)); +} + +#endif /* CONFIG_SMP */ + +static void switched_from_dl(struct rq *rq, struct task_struct *p) +{ + /* + * Start the deadline timer; if we switch back to dl before this we'll + * continue consuming our current CBS slice. If we stay outside of + * SCHED_DEADLINE until the deadline passes, the timer will reset the + * task. + */ + if (!start_dl_timer(p)) + __dl_clear_params(p); + + /* + * Since this might be the only -deadline task on the rq, + * this is the right place to try to pull some other one + * from an overloaded cpu, if any. + */ + if (!task_on_rq_queued(p) || rq->dl.dl_nr_running) + return; + + if (pull_dl_task(rq)) + resched_curr(rq); +} + +/* + * When switching to -deadline, we may overload the rq, then + * we try to push someone off, if possible. + */ +static void switched_to_dl(struct rq *rq, struct task_struct *p) +{ + int check_resched = 1; + + /* + * If p is throttled, don't consider the possibility + * of preempting rq->curr, the check will be done right + * after its runtime will get replenished. + */ + if (unlikely(p->dl.dl_throttled)) + return; + + if (task_on_rq_queued(p) && rq->curr != p) { +#ifdef CONFIG_SMP + if (rq->dl.overloaded && push_dl_task(rq) && rq != task_rq(p)) + /* Only reschedule if pushing failed */ + check_resched = 0; +#endif /* CONFIG_SMP */ + if (check_resched) { + if (dl_task(rq->curr)) + check_preempt_curr_dl(rq, p, 0); + else + resched_curr(rq); + } + } +} + +/* + * If the scheduling parameters of a -deadline task changed, + * a push or pull operation might be needed. + */ +static void prio_changed_dl(struct rq *rq, struct task_struct *p, + int oldprio) +{ + if (task_on_rq_queued(p) || rq->curr == p) { +#ifdef CONFIG_SMP + /* + * This might be too much, but unfortunately + * we don't have the old deadline value, and + * we can't argue if the task is increasing + * or lowering its prio, so... + */ + if (!rq->dl.overloaded) + pull_dl_task(rq); + + /* + * If we now have a earlier deadline task than p, + * then reschedule, provided p is still on this + * runqueue. + */ + if (dl_time_before(rq->dl.earliest_dl.curr, p->dl.deadline) && + rq->curr == p) + resched_curr(rq); +#else + /* + * Again, we don't know if p has a earlier + * or later deadline, so let's blindly set a + * (maybe not needed) rescheduling point. + */ + resched_curr(rq); +#endif /* CONFIG_SMP */ + } else + switched_to_dl(rq, p); +} + +const struct sched_class dl_sched_class = { + .next = &rt_sched_class, + .enqueue_task = enqueue_task_dl, + .dequeue_task = dequeue_task_dl, + .yield_task = yield_task_dl, + + .check_preempt_curr = check_preempt_curr_dl, + + .pick_next_task = pick_next_task_dl, + .put_prev_task = put_prev_task_dl, + +#ifdef CONFIG_SMP + .select_task_rq = select_task_rq_dl, + .set_cpus_allowed = set_cpus_allowed_dl, + .rq_online = rq_online_dl, + .rq_offline = rq_offline_dl, + .post_schedule = post_schedule_dl, + .task_woken = task_woken_dl, +#endif + + .set_curr_task = set_curr_task_dl, + .task_tick = task_tick_dl, + .task_fork = task_fork_dl, + .task_dead = task_dead_dl, + + .prio_changed = prio_changed_dl, + .switched_from = switched_from_dl, + .switched_to = switched_to_dl, + + .update_curr = update_curr_dl, +}; diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index a4e8ef3aca901..bde6dd63d6698 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "sched.h" @@ -70,7 +71,7 @@ static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group if (!se) { struct sched_avg *avg = &cpu_rq(cpu)->avg; P(avg->runnable_avg_sum); - P(avg->runnable_avg_period); + P(avg->avg_period); return; } @@ -93,7 +94,7 @@ static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group P(se->load.weight); #ifdef CONFIG_SMP P(se->avg.runnable_avg_sum); - P(se->avg.runnable_avg_period); + P(se->avg.avg_period); P(se->avg.load_avg_contrib); P(se->avg.decay_count); #endif @@ -124,7 +125,7 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) SEQ_printf(m, " "); SEQ_printf(m, "%15s %5d %9Ld.%06ld %9Ld %5d ", - p->comm, p->pid, + p->comm, task_pid_nr(p), SPLIT_NS(p->se.vruntime), (long long)(p->nvcsw + p->nivcsw), p->prio); @@ -137,6 +138,9 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) SEQ_printf(m, "%15Ld %15Ld %15Ld.%06ld %15Ld.%06ld %15Ld.%06ld", 0LL, 0LL, 0LL, 0L, 0LL, 0L, 0LL, 0L); #endif +#ifdef CONFIG_NUMA_BALANCING + SEQ_printf(m, " %d", task_node(p)); +#endif #ifdef CONFIG_CGROUP_SCHED SEQ_printf(m, " %s", task_group_path(task_group(p))); #endif @@ -147,7 +151,6 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) { struct task_struct *g, *p; - unsigned long flags; SEQ_printf(m, "\nrunnable tasks:\n" @@ -156,16 +159,14 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) "------------------------------------------------------" "----------------------------------------------------\n"); - read_lock_irqsave(&tasklist_lock, flags); - - do_each_thread(g, p) { - if (!p->on_rq || task_cpu(p) != rq_cpu) + rcu_read_lock(); + for_each_process_thread(g, p) { + if (task_cpu(p) != rq_cpu) continue; print_task(m, rq, p); - } while_each_thread(g, p); - - read_unlock_irqrestore(&tasklist_lock, flags); + } + rcu_read_unlock(); } void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) @@ -209,29 +210,24 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) cfs_rq->nr_spread_over); SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running); SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight); -#ifdef CONFIG_CFS_BANDWIDTH - SEQ_printf(m, " .%-30s: %d\n", "tg->cfs_bandwidth.timer_active", - cfs_rq->tg->cfs_bandwidth.timer_active); - SEQ_printf(m, " .%-30s: %d\n", "throttled", - cfs_rq->throttled); - SEQ_printf(m, " .%-30s: %d\n", "throttle_count", - cfs_rq->throttle_count); -#endif -#ifdef CONFIG_FAIR_GROUP_SCHED #ifdef CONFIG_SMP - SEQ_printf(m, " .%-30s: %lld\n", "runnable_load_avg", + SEQ_printf(m, " .%-30s: %ld\n", "runnable_load_avg", cfs_rq->runnable_load_avg); - SEQ_printf(m, " .%-30s: %lld\n", "blocked_load_avg", + SEQ_printf(m, " .%-30s: %ld\n", "blocked_load_avg", cfs_rq->blocked_load_avg); - SEQ_printf(m, " .%-30s: %lld\n", "tg_load_avg", - (unsigned long long)atomic64_read(&cfs_rq->tg->load_avg)); - SEQ_printf(m, " .%-30s: %lld\n", "tg_load_contrib", + SEQ_printf(m, " .%-30s: %ld\n", "utilization_load_avg", + cfs_rq->utilization_load_avg); +#ifdef CONFIG_FAIR_GROUP_SCHED + SEQ_printf(m, " .%-30s: %ld\n", "tg_load_contrib", cfs_rq->tg_load_contrib); SEQ_printf(m, " .%-30s: %d\n", "tg_runnable_contrib", cfs_rq->tg_runnable_contrib); + SEQ_printf(m, " .%-30s: %ld\n", "tg_load_avg", + atomic_long_read(&cfs_rq->tg->load_avg)); SEQ_printf(m, " .%-30s: %d\n", "tg->runnable_avg", atomic_read(&cfs_rq->tg->runnable_avg)); #endif +#endif #ifdef CONFIG_CFS_BANDWIDTH SEQ_printf(m, " .%-30s: %d\n", "tg->cfs_bandwidth.timer_active", cfs_rq->tg->cfs_bandwidth.timer_active); @@ -239,18 +235,9 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) cfs_rq->throttled); SEQ_printf(m, " .%-30s: %d\n", "throttle_count", cfs_rq->throttle_count); - SEQ_printf(m, " .%-30s: %d\n", "runtime_enabled", - cfs_rq->runtime_enabled); -#ifdef CONFIG_SCHED_HMP - SEQ_printf(m, " .%-30s: %d\n", "nr_big_tasks", - cfs_rq->hmp_stats.nr_big_tasks); - SEQ_printf(m, " .%-30s: %d\n", "nr_small_tasks", - cfs_rq->hmp_stats.nr_small_tasks); - SEQ_printf(m, " .%-30s: %llu\n", "cumulative_runnable_avg", - cfs_rq->hmp_stats.cumulative_runnable_avg); -#endif #endif +#ifdef CONFIG_FAIR_GROUP_SCHED print_cfs_group_stats(m, cpu, cfs_rq->tg); #endif } @@ -313,32 +300,13 @@ do { \ P(nr_load_updates); P(nr_uninterruptible); PN(next_balance); - P(curr->pid); + SEQ_printf(m, " .%-30s: %ld\n", "curr->pid", (long)(task_pid_nr(rq->curr))); PN(clock); P(cpu_load[0]); P(cpu_load[1]); P(cpu_load[2]); P(cpu_load[3]); P(cpu_load[4]); -#ifdef CONFIG_SMP - P(cpu_power); -#endif -#ifdef CONFIG_SCHED_HMP - P(mostly_idle_load); - P(mostly_idle_nr_run); - P(load_scale_factor); - P(capacity); - P(max_possible_capacity); - P(efficiency); - P(cur_freq); - P(max_freq); -#endif -#ifdef CONFIG_SCHED_HMP - P(hmp_stats.nr_big_tasks); - P(hmp_stats.nr_small_tasks); - SEQ_printf(m, " .%-30s: %llu\n", "hmp_stats.cumulative_runnable_avg", - rq->hmp_stats.cumulative_runnable_avg); -#endif #undef P #undef PN @@ -352,6 +320,7 @@ do { \ P(sched_goidle); #ifdef CONFIG_SMP P64(avg_idle); + P64(max_idle_balance_cost); #endif P(ttwu_count); @@ -364,9 +333,7 @@ do { \ print_cfs_stats(m, cpu); print_rt_stats(m, cpu); - rcu_read_lock(); print_rq(m, rq, cpu); - rcu_read_unlock(); spin_unlock_irqrestore(&sched_debug_lock, flags); SEQ_printf(m, "\n"); } @@ -388,7 +355,7 @@ static void sched_debug_header(struct seq_file *m) cpu_clk = local_clock(); local_irq_restore(flags); - SEQ_printf(m, "Sched Debug Version: v0.10, %s %.*s\n", + SEQ_printf(m, "Sched Debug Version: v0.11, %s %.*s\n", init_utsname()->release, (int)strcspn(init_utsname()->version, " "), init_utsname()->version); @@ -402,7 +369,7 @@ static void sched_debug_header(struct seq_file *m) PN(cpu_clk); P(jiffies); #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK - P(sched_clock_stable); + P(sched_clock_stable()); #endif #undef PN #undef P @@ -419,17 +386,6 @@ static void sched_debug_header(struct seq_file *m) PN(sysctl_sched_wakeup_granularity); P(sysctl_sched_child_runs_first); P(sysctl_sched_features); -#ifdef CONFIG_SCHED_HMP - P(sched_small_task); - P(sched_upmigrate); - P(sched_downmigrate); - P(sched_init_task_load_windows); - P(sched_init_task_load_pelt); - P(min_capacity); - P(max_capacity); - P(sched_use_pelt); - P(sched_ravg_window); -#endif #undef PN #undef P @@ -452,7 +408,6 @@ static int sched_debug_show(struct seq_file *m, void *v) return 0; } -#ifdef CONFIG_SYSRQ_SCHED_DEBUG void sysrq_sched_debug_show(void) { int cpu; @@ -462,7 +417,6 @@ void sysrq_sched_debug_show(void) print_cpu(NULL, cpu); } -#endif /* * This itererator needs some explanation. @@ -544,25 +498,73 @@ static int __init init_sched_debug_procfs(void) __initcall(init_sched_debug_procfs); +#define __P(F) \ + SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)F) +#define P(F) \ + SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)p->F) +#define __PN(F) \ + SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F)) +#define PN(F) \ + SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F)) + + +static void sched_show_numa(struct task_struct *p, struct seq_file *m) +{ +#ifdef CONFIG_NUMA_BALANCING + struct mempolicy *pol; + int node, i; + + if (p->mm) + P(mm->numa_scan_seq); + + task_lock(p); + pol = p->mempolicy; + if (pol && !(pol->flags & MPOL_F_MORON)) + pol = NULL; + mpol_get(pol); + task_unlock(p); + + SEQ_printf(m, "numa_migrations, %ld\n", xchg(&p->numa_pages_migrated, 0)); + + for_each_online_node(node) { + for (i = 0; i < 2; i++) { + unsigned long nr_faults = -1; + int cpu_current, home_node; + + if (p->numa_faults_memory) + nr_faults = p->numa_faults_memory[2*node + i]; + + cpu_current = !i ? (task_node(p) == node) : + (pol && node_isset(node, pol->v.nodes)); + + home_node = (p->numa_preferred_nid == node); + + SEQ_printf(m, "numa_faults_memory, %d, %d, %d, %d, %ld\n", + i, node, cpu_current, home_node, nr_faults); + } + } + + mpol_put(pol); +#endif +} + void proc_sched_show_task(struct task_struct *p, struct seq_file *m) { unsigned long nr_switches; - unsigned int load_avg; - - load_avg = pct_task_load(p); - SEQ_printf(m, "%s (%d, #threads: %d)\n", p->comm, p->pid, + SEQ_printf(m, "%s (%d, #threads: %d)\n", p->comm, task_pid_nr(p), get_nr_threads(p)); SEQ_printf(m, - "---------------------------------------------------------\n"); + "---------------------------------------------------------" + "----------\n"); #define __P(F) \ - SEQ_printf(m, "%-35s:%21Ld\n", #F, (long long)F) + SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)F) #define P(F) \ - SEQ_printf(m, "%-35s:%21Ld\n", #F, (long long)p->F) + SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)p->F) #define __PN(F) \ - SEQ_printf(m, "%-35s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F)) + SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F)) #define PN(F) \ - SEQ_printf(m, "%-35s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F)) + SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F)) PN(se.exec_start); PN(se.vruntime); @@ -599,16 +601,6 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) P(se.statistics.nr_wakeups_passive); P(se.statistics.nr_wakeups_idle); -#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED) - __P(load_avg); -#ifdef CONFIG_SCHED_HMP - P(ravg.demand); - P(se.avg.runnable_avg_sum_scaled); -#endif - P(se.avg.runnable_avg_sum); - P(se.avg.runnable_avg_period); -#endif - { u64 avg_atom, avg_per_cpu; @@ -631,12 +623,20 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) } #endif __P(nr_switches); - SEQ_printf(m, "%-35s:%21Ld\n", + SEQ_printf(m, "%-45s:%21Ld\n", "nr_voluntary_switches", (long long)p->nvcsw); - SEQ_printf(m, "%-35s:%21Ld\n", + SEQ_printf(m, "%-45s:%21Ld\n", "nr_involuntary_switches", (long long)p->nivcsw); P(se.load.weight); +#ifdef CONFIG_SMP + P(se.avg.runnable_avg_sum); + P(se.avg.running_avg_sum); + P(se.avg.avg_period); + P(se.avg.load_avg_contrib); + P(se.avg.utilization_avg_contrib); + P(se.avg.decay_count); +#endif P(policy); P(prio); #undef PN @@ -650,9 +650,11 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) t0 = cpu_clock(this_cpu); t1 = cpu_clock(this_cpu); - SEQ_printf(m, "%-35s:%21Ld\n", + SEQ_printf(m, "%-45s:%21Ld\n", "clock-delta", (long long)(t1-t0)); } + + sched_show_numa(p, m); } void proc_sched_set_task(struct task_struct *p) diff --git a/kernel/sched/energy.c b/kernel/sched/energy.c new file mode 100644 index 0000000000000..b0656b7a93e33 --- /dev/null +++ b/kernel/sched/energy.c @@ -0,0 +1,124 @@ +/* + * Obtain energy cost data from DT and populate relevant scheduler data + * structures. + * + * Copyright (C) 2015 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#define pr_fmt(fmt) "sched-energy: " fmt + +#define DEBUG + +#include +#include +#include +#include +#include +#include + +struct sched_group_energy *sge_array[NR_CPUS][NR_SD_LEVELS]; + +static void free_resources(void) +{ + int cpu, sd_level; + struct sched_group_energy *sge; + + for_each_possible_cpu(cpu) { + for_each_possible_sd_level(sd_level) { + sge = sge_array[cpu][sd_level]; + if (sge) { + kfree(sge->cap_states); + kfree(sge->idle_states); + kfree(sge); + } + } + } +} + +void init_sched_energy_costs(void) +{ + struct device_node *cn, *cp; + struct capacity_state *cap_states; + struct idle_state *idle_states; + struct sched_group_energy *sge; + const struct property *prop; + int sd_level, i, nstates, cpu; + const __be32 *val; + + for_each_possible_cpu(cpu) { + cn = of_get_cpu_node(cpu, NULL); + if (!cn) { + pr_warn("CPU device node missing for CPU %d\n", cpu); + return; + } + + if (!of_find_property(cn, "sched-energy-costs", NULL)) { + pr_warn("CPU device node has no sched-energy-costs\n"); + return; + } + + for_each_possible_sd_level(sd_level) { + cp = of_parse_phandle(cn, "sched-energy-costs", sd_level); + if (!cp) + break; + + prop = of_find_property(cp, "busy-cost-data", NULL); + if (!prop || !prop->value) { + pr_warn("No busy-cost data, skipping sched_energy init\n"); + goto out; + } + + sge = kcalloc(1, sizeof(struct sched_group_energy), + GFP_NOWAIT); + + nstates = (prop->length / sizeof(u32)) / 2; + cap_states = kcalloc(nstates, + sizeof(struct capacity_state), + GFP_NOWAIT); + + for (i = 0, val = prop->value; i < nstates; i++) { + cap_states[i].cap = be32_to_cpup(val++); + cap_states[i].power = be32_to_cpup(val++); + } + + sge->nr_cap_states = nstates; + sge->cap_states = cap_states; + + prop = of_find_property(cp, "idle-cost-data", NULL); + if (!prop || !prop->value) { + pr_warn("No idle-cost data, skipping sched_energy init\n"); + goto out; + } + + nstates = (prop->length / sizeof(u32)); + idle_states = kcalloc(nstates, + sizeof(struct idle_state), + GFP_NOWAIT); + + for (i = 0, val = prop->value; i < nstates; i++) + idle_states[i].power = be32_to_cpup(val++); + + sge->nr_idle_states = nstates; + sge->idle_states = idle_states; + + sge_array[cpu][sd_level] = sge; + } + } + + pr_info("Sched-energy-costs installed from DT\n"); + return; + +out: + free_resources(); +} diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 60d836e5ecb88..bc7685196b09a 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -33,6 +34,12 @@ #include #include "sched.h" +#include "tune.h" + +#ifndef TJK_HMP +unsigned int __read_mostly sysctl_sched_wake_to_idle = 0; +#endif + /* * Targeted preemption latency for CPU-bound tasks: @@ -49,6 +56,9 @@ unsigned int sysctl_sched_latency = 6000000ULL; unsigned int normalized_sysctl_sched_latency = 6000000ULL; +unsigned int sysctl_sched_is_big_little = 0; +unsigned int sysctl_sched_sync_hint_enable = 1; + /* * The initial- and re-scaling of tunables is configurable * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) @@ -79,32 +89,6 @@ static unsigned int sched_nr_latency = 8; */ unsigned int sysctl_sched_child_runs_first __read_mostly; -#ifdef VENDOR_EDIT -enum thermal_aware_scheduling { - SCHED_TA_DISABLE, - SCHED_TA_THERMAL_ONLY, - SCHED_TA_ALWAYS_ON, -}; -/* - * Thermal-aware scheduling, prefer running on small cluster - * when task loading on big cluster is below certain threshold - * - * Options: - * SCHED_TA_DISABLE - Disable TA scheduling - * SCHED_TA_THERMAL_ONLY - Only enable when under thermal constraint - * SCHED_TA_ALWAYS_ON - Always enable TA scheduling - */ -unsigned int __read_mostly sysctl_thermal_aware_scheduling = SCHED_TA_THERMAL_ONLY; -#endif - -/* - * Controls whether, when SD_SHARE_PKG_RESOURCES is on, if all - * tasks go to idle CPUs when woken. If this is off, note that the - * per-task flag PF_WAKE_UP_IDLE can still cause a task to go to an - * idle CPU upon being woken. - */ -unsigned int __read_mostly sysctl_sched_wake_to_idle; - /* * SCHED_OTHER wake-up granularity. * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) @@ -139,6 +123,24 @@ unsigned int __read_mostly sysctl_sched_shares_window = 10000000UL; unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; #endif +static inline void update_load_add(struct load_weight *lw, unsigned long inc) +{ + lw->weight += inc; + lw->inv_weight = 0; +} + +static inline void update_load_sub(struct load_weight *lw, unsigned long dec) +{ + lw->weight -= dec; + lw->inv_weight = 0; +} + +static inline void update_load_set(struct load_weight *lw, unsigned long w) +{ + lw->weight = w; + lw->inv_weight = 0; +} + /* * Increase the granularity value when there are more CPUs, * because with more CPUs the 'effective latency' as visible @@ -186,64 +188,78 @@ void sched_init_granularity(void) update_sysctl(); } -#if BITS_PER_LONG == 32 -# define WMULT_CONST (~0UL) -#else -# define WMULT_CONST (1UL << 32) -#endif - +#define WMULT_CONST (~0U) #define WMULT_SHIFT 32 -/* - * Shift right and round: - */ -#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y)) +static void __update_inv_weight(struct load_weight *lw) +{ + unsigned long w; + + if (likely(lw->inv_weight)) + return; + + w = scale_load_down(lw->weight); + + if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) + lw->inv_weight = 1; + else if (unlikely(!w)) + lw->inv_weight = WMULT_CONST; + else + lw->inv_weight = WMULT_CONST / w; +} + +static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift) +{ + u32 ah, al; + u64 ret; + + al = a; + ah = a >> 32; + + ret = ((u64)al * mul) >> shift; + if (ah) + ret += ((u64)ah * mul) << (32 - shift); + + return ret; +} /* - * delta *= weight / lw + * delta_exec * weight / lw.weight + * OR + * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT + * + * Either weight := NICE_0_LOAD and lw \e prio_to_wmult[], in which case + * we're guaranteed shift stays positive because inv_weight is guaranteed to + * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22. + * + * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus + * weight/lw.weight <= 1, and therefore our shift will also be positive. */ -static unsigned long -calc_delta_mine(unsigned long delta_exec, unsigned long weight, - struct load_weight *lw) +static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) { - u64 tmp; - - /* - * weight can be less than 2^SCHED_LOAD_RESOLUTION for task group sched - * entities since MIN_SHARES = 2. Treat weight as 1 if less than - * 2^SCHED_LOAD_RESOLUTION. - */ - if (likely(weight > (1UL << SCHED_LOAD_RESOLUTION))) - tmp = (u64)delta_exec * scale_load_down(weight); - else - tmp = (u64)delta_exec; + u64 fact = scale_load_down(weight); + int shift = WMULT_SHIFT; - if (!lw->inv_weight) { - unsigned long w = scale_load_down(lw->weight); + __update_inv_weight(lw); - if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) - lw->inv_weight = 1; - else if (unlikely(!w)) - lw->inv_weight = WMULT_CONST; - else - lw->inv_weight = WMULT_CONST / w; + if (unlikely(fact >> 32)) { + while (fact >> 32) { + fact >>= 1; + shift--; + } } - /* - * Check whether we'd overflow the 64-bit multiplication: - */ - if (unlikely(tmp > WMULT_CONST)) - tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight, - WMULT_SHIFT/2); - else - tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT); + /* hint to use a 32x32->64 mul */ + fact = (u64)(u32)fact * lw->inv_weight; + + while (fact >> 32) { + fact >>= 1; + shift--; + } - return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX); + return mul_u64_u32_shr(delta_exec, fact, shift); } -#ifdef CONFIG_SMP -static int active_load_balance_cpu_stop(void *data); -#endif const struct sched_class fair_sched_class; @@ -331,13 +347,13 @@ static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) list_for_each_entry_rcu(cfs_rq, &rq->leaf_cfs_rq_list, leaf_cfs_rq_list) /* Do the two (enqueued) entities belong to the same group ? */ -static inline int +static inline struct cfs_rq * is_same_group(struct sched_entity *se, struct sched_entity *pse) { if (se->cfs_rq == pse->cfs_rq) - return 1; + return se->cfs_rq; - return 0; + return NULL; } static inline struct sched_entity *parent_entity(struct sched_entity *se) @@ -345,17 +361,6 @@ static inline struct sched_entity *parent_entity(struct sched_entity *se) return se->parent; } -/* return depth at which a sched entity is present in the hierarchy */ -static inline int depth_se(struct sched_entity *se) -{ - int depth = 0; - - for_each_sched_entity(se) - depth++; - - return depth; -} - static void find_matching_se(struct sched_entity **se, struct sched_entity **pse) { @@ -369,8 +374,8 @@ find_matching_se(struct sched_entity **se, struct sched_entity **pse) */ /* First walk up until both entities are at same depth */ - se_depth = depth_se(*se); - pse_depth = depth_se(*pse); + se_depth = (*se)->depth; + pse_depth = (*pse)->depth; while (se_depth > pse_depth) { se_depth--; @@ -435,12 +440,6 @@ static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) #define for_each_leaf_cfs_rq(rq, cfs_rq) \ for (cfs_rq = &rq->cfs; cfs_rq; cfs_rq = NULL) -static inline int -is_same_group(struct sched_entity *se, struct sched_entity *pse) -{ - return 1; -} - static inline struct sched_entity *parent_entity(struct sched_entity *se) { return NULL; @@ -454,7 +453,7 @@ find_matching_se(struct sched_entity **se, struct sched_entity **pse) #endif /* CONFIG_FAIR_GROUP_SCHED */ static __always_inline -void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, unsigned long delta_exec); +void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); /************************************************************** * Scheduling class tree data structure manipulation methods: @@ -623,11 +622,10 @@ int sched_proc_update_handler(struct ctl_table *table, int write, /* * delta /= w */ -static inline unsigned long -calc_delta_fair(unsigned long delta, struct sched_entity *se) +static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) { if (unlikely(se->load.weight != NICE_0_LOAD)) - delta = calc_delta_mine(delta, NICE_0_LOAD, &se->load); + delta = __calc_delta(delta, NICE_0_LOAD, &se->load); return delta; } @@ -642,15 +640,10 @@ calc_delta_fair(unsigned long delta, struct sched_entity *se) */ static u64 __sched_period(unsigned long nr_running) { - u64 period = sysctl_sched_latency; - unsigned long nr_latency = sched_nr_latency; - - if (unlikely(nr_running > nr_latency)) { - period = sysctl_sched_min_granularity; - period *= nr_running; - } - - return period; + if (unlikely(nr_running > sched_nr_latency)) + return nr_running * sysctl_sched_min_granularity; + else + return sysctl_sched_latency; } /* @@ -676,7 +669,7 @@ static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) update_load_add(&lw, se->load.weight); load = &lw; } - slice = calc_delta_mine(slice, se->load.weight, load); + slice = __calc_delta(slice, se->load.weight, load); } return slice; } @@ -691,48 +684,57 @@ static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se) return calc_delta_fair(sched_slice(cfs_rq, se), se); } -/* - * Update the current task's runtime statistics. Skip current tasks that - * are not in our scheduling class. - */ -static inline void -__update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr, - unsigned long delta_exec) -{ - unsigned long delta_exec_weighted; +#ifdef CONFIG_SMP +static int select_idle_sibling(struct task_struct *p, int cpu); +static unsigned long task_h_load(struct task_struct *p); - schedstat_set(curr->statistics.exec_max, - max((u64)delta_exec, curr->statistics.exec_max)); +static inline void __update_task_entity_contrib(struct sched_entity *se); +static inline void __update_task_entity_utilization(struct sched_entity *se); - curr->sum_exec_runtime += delta_exec; - schedstat_add(cfs_rq, exec_clock, delta_exec); - delta_exec_weighted = calc_delta_fair(delta_exec, curr); +/* Give new task start runnable values to heavy its load in infant time */ +void init_task_runnable_average(struct task_struct *p) +{ + u32 slice; - curr->vruntime += delta_exec_weighted; - update_min_vruntime(cfs_rq); + slice = sched_slice(task_cfs_rq(p), &p->se) >> 10; + p->se.avg.runnable_avg_sum = p->se.avg.running_avg_sum = slice; + p->se.avg.avg_period = slice; + __update_task_entity_contrib(&p->se); + __update_task_entity_utilization(&p->se); +} +#else +void init_task_runnable_average(struct task_struct *p) +{ } +#endif +/* + * Update the current task's runtime statistics. + */ static void update_curr(struct cfs_rq *cfs_rq) { struct sched_entity *curr = cfs_rq->curr; - u64 now = rq_of(cfs_rq)->clock_task; - unsigned long delta_exec; + u64 now = rq_clock_task(rq_of(cfs_rq)); + u64 delta_exec; if (unlikely(!curr)) return; - /* - * Get the amount of time the current task was running - * since the last time we changed load (this cannot - * overflow on 32 bits): - */ - delta_exec = (unsigned long)(now - curr->exec_start); - if (!delta_exec) + delta_exec = now - curr->exec_start; + if (unlikely((s64)delta_exec <= 0)) return; - __update_curr(cfs_rq, curr, delta_exec); curr->exec_start = now; + schedstat_set(curr->statistics.exec_max, + max(delta_exec, curr->statistics.exec_max)); + + curr->sum_exec_runtime += delta_exec; + schedstat_add(cfs_rq, exec_clock, delta_exec); + + curr->vruntime += calc_delta_fair(delta_exec, curr); + update_min_vruntime(cfs_rq); + if (entity_is_task(curr)) { struct task_struct *curtask = task_of(curr); @@ -744,10 +746,15 @@ static void update_curr(struct cfs_rq *cfs_rq) account_cfs_rq_runtime(cfs_rq, delta_exec); } +static void update_curr_fair(struct rq *rq) +{ + update_curr(cfs_rq_of(&rq->curr->se)); +} + static inline void update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se) { - schedstat_set(se->statistics.wait_start, rq_of(cfs_rq)->clock); + schedstat_set(se->statistics.wait_start, rq_clock(rq_of(cfs_rq))); } /* @@ -767,14 +774,14 @@ static void update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) { schedstat_set(se->statistics.wait_max, max(se->statistics.wait_max, - rq_of(cfs_rq)->clock - se->statistics.wait_start)); + rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start)); schedstat_set(se->statistics.wait_count, se->statistics.wait_count + 1); schedstat_set(se->statistics.wait_sum, se->statistics.wait_sum + - rq_of(cfs_rq)->clock - se->statistics.wait_start); + rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start); #ifdef CONFIG_SCHEDSTATS if (entity_is_task(se)) { trace_sched_stat_wait(task_of(se), - rq_of(cfs_rq)->clock - se->statistics.wait_start); + rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start); } #endif schedstat_set(se->statistics.wait_start, 0); @@ -800,7 +807,7 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) /* * We are starting a new run period: */ - se->exec_start = rq_of(cfs_rq)->clock_task; + se->exec_start = rq_clock_task(rq_of(cfs_rq)); } /************************************************** @@ -809,11 +816,12 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) #ifdef CONFIG_NUMA_BALANCING /* - * numa task sample period in ms + * Approximate time to scan a full NUMA task in ms. The task scan period is + * calculated based on the tasks virtual memory size and + * numa_balancing_scan_size. */ -unsigned int sysctl_numa_balancing_scan_period_min = 100; -unsigned int sysctl_numa_balancing_scan_period_max = 100*50; -unsigned int sysctl_numa_balancing_scan_period_reset = 100*600; +unsigned int sysctl_numa_balancing_scan_period_min = 1000; +unsigned int sysctl_numa_balancing_scan_period_max = 60000; /* Portion of address space to scan in MB */ unsigned int sysctl_numa_balancing_scan_size = 256; @@ -821,2262 +829,1427 @@ unsigned int sysctl_numa_balancing_scan_size = 256; /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */ unsigned int sysctl_numa_balancing_scan_delay = 1000; -static void task_numa_placement(struct task_struct *p) +static unsigned int task_nr_scan_windows(struct task_struct *p) { - int seq; + unsigned long rss = 0; + unsigned long nr_scan_pages; - if (!p->mm) /* for example, ksmd faulting in a user's mm */ - return; - seq = ACCESS_ONCE(p->mm->numa_scan_seq); - if (p->numa_scan_seq == seq) - return; - p->numa_scan_seq = seq; + /* + * Calculations based on RSS as non-present and empty pages are skipped + * by the PTE scanner and NUMA hinting faults should be trapped based + * on resident pages + */ + nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT); + rss = get_mm_rss(p->mm); + if (!rss) + rss = nr_scan_pages; - /* FIXME: Scheduling placement policy hints go here */ + rss = round_up(rss, nr_scan_pages); + return rss / nr_scan_pages; } -/* - * Got a PROT_NONE fault for a page on @node. - */ -void task_numa_fault(int node, int pages, bool migrated) +/* For sanitys sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */ +#define MAX_SCAN_WINDOW 2560 + +static unsigned int task_scan_min(struct task_struct *p) { - struct task_struct *p = current; + unsigned int scan_size = ACCESS_ONCE(sysctl_numa_balancing_scan_size); + unsigned int scan, floor; + unsigned int windows = 1; - if (!sched_feat_numa(NUMA)) - return; + if (scan_size < MAX_SCAN_WINDOW) + windows = MAX_SCAN_WINDOW / scan_size; + floor = 1000 / windows; - /* FIXME: Allocate task-specific structure for placement policy here */ + scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); + return max_t(unsigned int, floor, scan); +} + +static unsigned int task_scan_max(struct task_struct *p) +{ + unsigned int smin = task_scan_min(p); + unsigned int smax; + + /* Watch for min being lower than max due to floor calculations */ + smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); + return max(smin, smax); +} + +static void account_numa_enqueue(struct rq *rq, struct task_struct *p) +{ + rq->nr_numa_running += (p->numa_preferred_nid != -1); + rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); +} + +static void account_numa_dequeue(struct rq *rq, struct task_struct *p) +{ + rq->nr_numa_running -= (p->numa_preferred_nid != -1); + rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); +} + +struct numa_group { + atomic_t refcount; + spinlock_t lock; /* nr_tasks, tasks */ + int nr_tasks; + pid_t gid; + struct list_head task_list; + + struct rcu_head rcu; + nodemask_t active_nodes; + unsigned long total_faults; /* - * If pages are properly placed (did not migrate) then scan slower. - * This is reset periodically in case of phase changes + * Faults_cpu is used to decide whether memory should move + * towards the CPU. As a consequence, these stats are weighted + * more by CPU use than by memory faults. */ - if (!migrated) - p->numa_scan_period = min(sysctl_numa_balancing_scan_period_max, - p->numa_scan_period + jiffies_to_msecs(10)); + unsigned long *faults_cpu; + unsigned long faults[0]; +}; - task_numa_placement(p); +/* Shared or private faults. */ +#define NR_NUMA_HINT_FAULT_TYPES 2 + +/* Memory and CPU locality */ +#define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2) + +/* Averaged statistics, and temporary buffers. */ +#define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2) + +pid_t task_numa_group_id(struct task_struct *p) +{ + return p->numa_group ? p->numa_group->gid : 0; } -static void reset_ptenuma_scan(struct task_struct *p) +static inline int task_faults_idx(int nid, int priv) { - ACCESS_ONCE(p->mm->numa_scan_seq)++; - p->mm->numa_scan_offset = 0; + return NR_NUMA_HINT_FAULT_TYPES * nid + priv; +} + +static inline unsigned long task_faults(struct task_struct *p, int nid) +{ + if (!p->numa_faults_memory) + return 0; + + return p->numa_faults_memory[task_faults_idx(nid, 0)] + + p->numa_faults_memory[task_faults_idx(nid, 1)]; +} + +static inline unsigned long group_faults(struct task_struct *p, int nid) +{ + if (!p->numa_group) + return 0; + + return p->numa_group->faults[task_faults_idx(nid, 0)] + + p->numa_group->faults[task_faults_idx(nid, 1)]; +} + +static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) +{ + return group->faults_cpu[task_faults_idx(nid, 0)] + + group->faults_cpu[task_faults_idx(nid, 1)]; } /* - * The expensive part of numa migration is done from task_work context. - * Triggered from task_tick_numa(). + * These return the fraction of accesses done by a particular task, or + * task group, on a particular numa node. The group weight is given a + * larger multiplier, in order to group tasks together that are almost + * evenly spread out between numa nodes. */ -void task_numa_work(struct callback_head *work) +static inline unsigned long task_weight(struct task_struct *p, int nid) { - unsigned long migrate, next_scan, now = jiffies; - struct task_struct *p = current; - struct mm_struct *mm = p->mm; - struct vm_area_struct *vma; - unsigned long start, end; - long pages; + unsigned long total_faults; - WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work)); + if (!p->numa_faults_memory) + return 0; + + total_faults = p->total_numa_faults; + + if (!total_faults) + return 0; + + return 1000 * task_faults(p, nid) / total_faults; +} + +static inline unsigned long group_weight(struct task_struct *p, int nid) +{ + if (!p->numa_group || !p->numa_group->total_faults) + return 0; + + return 1000 * group_faults(p, nid) / p->numa_group->total_faults; +} + +bool should_numa_migrate_memory(struct task_struct *p, struct page * page, + int src_nid, int dst_cpu) +{ + struct numa_group *ng = p->numa_group; + int dst_nid = cpu_to_node(dst_cpu); + int last_cpupid, this_cpupid; + + this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); - work->next = work; /* protect against double add */ /* - * Who cares about NUMA placement when they're dying. + * Multi-stage node selection is used in conjunction with a periodic + * migration fault to build a temporal task<->page relation. By using + * a two-stage filter we remove short/unlikely relations. * - * NOTE: make sure not to dereference p->mm before this check, - * exit_task_work() happens _after_ exit_mm() so we could be called - * without p->mm even though we still had it when we enqueued this - * work. + * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate + * a task's usage of a particular page (n_p) per total usage of this + * page (n_t) (in a given time-span) to a probability. + * + * Our periodic faults will sample this probability and getting the + * same result twice in a row, given these samples are fully + * independent, is then given by P(n)^2, provided our sample period + * is sufficiently short compared to the usage pattern. + * + * This quadric squishes small probabilities, making it less likely we + * act on an unlikely task<->page relation. */ - if (p->flags & PF_EXITING) - return; + last_cpupid = page_cpupid_xchg_last(page, this_cpupid); + if (!cpupid_pid_unset(last_cpupid) && + cpupid_to_nid(last_cpupid) != dst_nid) + return false; - /* - * We do not care about task placement until a task runs on a node - * other than the first one used by the address space. This is - * largely because migrations are driven by what CPU the task - * is running on. If it's never scheduled on another node, it'll - * not migrate so why bother trapping the fault. - */ - if (mm->first_nid == NUMA_PTE_SCAN_INIT) - mm->first_nid = numa_node_id(); - if (mm->first_nid != NUMA_PTE_SCAN_ACTIVE) { - /* Are we running on a new node yet? */ - if (numa_node_id() == mm->first_nid && - !sched_feat_numa(NUMA_FORCE)) - return; + /* Always allow migrate on private faults */ + if (cpupid_match_pid(p, last_cpupid)) + return true; - mm->first_nid = NUMA_PTE_SCAN_ACTIVE; - } + /* A shared fault, but p->numa_group has not been set up yet. */ + if (!ng) + return true; /* - * Reset the scan period if enough time has gone by. Objective is that - * scanning will be reduced if pages are properly placed. As tasks - * can enter different phases this needs to be re-examined. Lacking - * proper tracking of reference behaviour, this blunt hammer is used. + * Do not migrate if the destination is not a node that + * is actively used by this numa group. */ - migrate = mm->numa_next_reset; - if (time_after(now, migrate)) { - p->numa_scan_period = sysctl_numa_balancing_scan_period_min; - next_scan = now + msecs_to_jiffies(sysctl_numa_balancing_scan_period_reset); - xchg(&mm->numa_next_reset, next_scan); - } + if (!node_isset(dst_nid, ng->active_nodes)) + return false; /* - * Enforce maximal scan/migration frequency.. + * Source is a node that is not actively used by this + * numa group, while the destination is. Migrate. */ - migrate = mm->numa_next_scan; - if (time_before(now, migrate)) - return; - - if (p->numa_scan_period == 0) - p->numa_scan_period = sysctl_numa_balancing_scan_period_min; - - next_scan = now + msecs_to_jiffies(p->numa_scan_period); - if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate) - return; + if (!node_isset(src_nid, ng->active_nodes)) + return true; /* - * Do not set pte_numa if the current running node is rate-limited. - * This loses statistics on the fault but if we are unwilling to - * migrate to this node, it is less likely we can do useful work + * Both source and destination are nodes in active + * use by this numa group. Maximize memory bandwidth + * by migrating from more heavily used groups, to less + * heavily used ones, spreading the load around. + * Use a 1/4 hysteresis to avoid spurious page movement. */ - if (migrate_ratelimited(numa_node_id())) - return; + return group_faults(p, dst_nid) < (group_faults(p, src_nid) * 3 / 4); +} - start = mm->numa_scan_offset; - pages = sysctl_numa_balancing_scan_size; - pages <<= 20 - PAGE_SHIFT; /* MB in pages */ - if (!pages) - return; +static unsigned long weighted_cpuload(const int cpu); +static unsigned long source_load(int cpu, int type); +static unsigned long target_load(int cpu, int type); +static unsigned long capacity_of(int cpu); +static long effective_load(struct task_group *tg, int cpu, long wl, long wg); - down_read(&mm->mmap_sem); - vma = find_vma(mm, start); - if (!vma) { - reset_ptenuma_scan(p); - start = 0; - vma = mm->mmap; - } - for (; vma; vma = vma->vm_next) { - if (!vma_migratable(vma)) - continue; +/* Cached statistics for all CPUs within a node */ +struct numa_stats { + unsigned long nr_running; + unsigned long load; - /* Skip small VMAs. They are not likely to be of relevance */ - if (vma->vm_end - vma->vm_start < HPAGE_SIZE) - continue; + /* Total compute capacity of CPUs on a node */ + unsigned long compute_capacity; - /* - * Skip inaccessible VMAs to avoid any confusion between - * PROT_NONE and NUMA hinting ptes - */ - if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) - continue; + /* Approximate capacity in terms of runnable tasks on a node */ + unsigned long task_capacity; + int has_free_capacity; +}; - do { - start = max(start, vma->vm_start); - end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); - end = min(end, vma->vm_end); - pages -= change_prot_numa(vma, start, end); +/* + * XXX borrowed from update_sg_lb_stats + */ +static void update_numa_stats(struct numa_stats *ns, int nid) +{ + int smt, cpu, cpus = 0; + unsigned long capacity; - start = end; - if (pages <= 0) - goto out; - } while (end != vma->vm_end); + memset(ns, 0, sizeof(*ns)); + for_each_cpu(cpu, cpumask_of_node(nid)) { + struct rq *rq = cpu_rq(cpu); + + ns->nr_running += rq->nr_running; + ns->load += weighted_cpuload(cpu); + ns->compute_capacity += capacity_of(cpu); + + cpus++; } -out: /* - * It is possible to reach the end of the VMA list but the last few VMAs are - * not guaranteed to the vma_migratable. If they are not, we would find the - * !migratable VMA on the next scan but not reset the scanner to the start - * so check it now. - */ - if (vma) - mm->numa_scan_offset = start; - else - reset_ptenuma_scan(p); - up_read(&mm->mmap_sem); -} - -/* - * Drive the periodic memory faults.. - */ -void task_tick_numa(struct rq *rq, struct task_struct *curr) -{ - struct callback_head *work = &curr->numa_work; - u64 period, now; - - /* - * We don't care about NUMA placement if we don't have memory. - */ - if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work) - return; - - /* - * Using runtime rather than walltime has the dual advantage that - * we (mostly) drive the selection from busy threads and that the - * task needs to have done some actual work before we bother with - * NUMA placement. - */ - now = curr->se.sum_exec_runtime; - period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; - - if (now - curr->node_stamp > period) { - if (!curr->node_stamp) - curr->numa_scan_period = sysctl_numa_balancing_scan_period_min; - curr->node_stamp = now; - - if (!time_before(jiffies, curr->mm->numa_next_scan)) { - init_task_work(work, task_numa_work); /* TODO: move this into sched_fork() */ - task_work_add(curr, work, true); - } - } -} -#else -static void task_tick_numa(struct rq *rq, struct task_struct *curr) -{ -} -#endif /* CONFIG_NUMA_BALANCING */ - -static void -account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) -{ - update_load_add(&cfs_rq->load, se->load.weight); - if (!parent_entity(se)) - update_load_add(&rq_of(cfs_rq)->load, se->load.weight); -#ifdef CONFIG_SMP - if (entity_is_task(se)) - list_add(&se->group_node, &rq_of(cfs_rq)->cfs_tasks); -#endif - cfs_rq->nr_running++; -} - -static void -account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) -{ - update_load_sub(&cfs_rq->load, se->load.weight); - if (!parent_entity(se)) - update_load_sub(&rq_of(cfs_rq)->load, se->load.weight); - if (entity_is_task(se)) - list_del_init(&se->group_node); - cfs_rq->nr_running--; -} - -#ifdef CONFIG_FAIR_GROUP_SCHED -# ifdef CONFIG_SMP -static inline long calc_tg_weight(struct task_group *tg, struct cfs_rq *cfs_rq) -{ - long tg_weight; - - /* - * Use this CPU's actual weight instead of the last load_contribution - * to gain a more accurate current total weight. See - * update_cfs_rq_load_contribution(). - */ - tg_weight = atomic64_read(&tg->load_avg); - tg_weight -= cfs_rq->tg_load_contrib; - tg_weight += cfs_rq->load.weight; - - return tg_weight; -} - -static long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg) -{ - long tg_weight, load, shares; - - tg_weight = calc_tg_weight(tg, cfs_rq); - load = cfs_rq->load.weight; - - shares = (tg->shares * load); - if (tg_weight) - shares /= tg_weight; - - if (shares < MIN_SHARES) - shares = MIN_SHARES; - if (shares > tg->shares) - shares = tg->shares; - - return shares; -} -# else /* CONFIG_SMP */ -static inline long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg) -{ - return tg->shares; -} -# endif /* CONFIG_SMP */ -static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, - unsigned long weight) -{ - if (se->on_rq) { - /* commit outstanding execution time */ - if (cfs_rq->curr == se) - update_curr(cfs_rq); - account_entity_dequeue(cfs_rq, se); - } - - update_load_set(&se->load, weight); - - if (se->on_rq) - account_entity_enqueue(cfs_rq, se); -} - -static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); - -static void update_cfs_shares(struct cfs_rq *cfs_rq) -{ - struct task_group *tg; - struct sched_entity *se; - long shares; - - tg = cfs_rq->tg; - se = tg->se[cpu_of(rq_of(cfs_rq))]; - if (!se || throttled_hierarchy(cfs_rq)) - return; -#ifndef CONFIG_SMP - if (likely(se->load.weight == tg->shares)) - return; -#endif - shares = calc_cfs_shares(cfs_rq, tg); - - reweight_entity(cfs_rq_of(se), se, shares); -} -#else /* CONFIG_FAIR_GROUP_SCHED */ -static inline void update_cfs_shares(struct cfs_rq *cfs_rq) -{ -} -#endif /* CONFIG_FAIR_GROUP_SCHED */ - -#ifdef CONFIG_SMP - -u32 sched_get_wake_up_idle(struct task_struct *p) -{ - u32 enabled = p->flags & PF_WAKE_UP_IDLE; - - return !!enabled; -} - -int sched_set_wake_up_idle(struct task_struct *p, int wake_up_idle) -{ - int enable = !!wake_up_idle; - - if (enable) - p->flags |= PF_WAKE_UP_IDLE; - else - p->flags &= ~PF_WAKE_UP_IDLE; - - return 0; -} - -#endif /* CONFIG_SMP */ - -/* Only depends on SMP, FAIR_GROUP_SCHED may be removed when useful in lb */ -#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED) -/* - * We choose a half-life close to 1 scheduling period. - * Note: The tables below are dependent on this value. - */ -#define LOAD_AVG_PERIOD 32 -#define LOAD_AVG_MAX 47742 /* maximum possible load avg */ -#define LOAD_AVG_MAX_N 345 /* number of full periods to produce LOAD_MAX_AVG */ - -/* Precomputed fixed inverse multiplies for multiplication by y^n */ -static const u32 runnable_avg_yN_inv[] = { - 0xffffffff, 0xfa83b2da, 0xf5257d14, 0xefe4b99a, 0xeac0c6e6, 0xe5b906e6, - 0xe0ccdeeb, 0xdbfbb796, 0xd744fcc9, 0xd2a81d91, 0xce248c14, 0xc9b9bd85, - 0xc5672a10, 0xc12c4cc9, 0xbd08a39e, 0xb8fbaf46, 0xb504f333, 0xb123f581, - 0xad583ee9, 0xa9a15ab4, 0xa5fed6a9, 0xa2704302, 0x9ef5325f, 0x9b8d39b9, - 0x9837f050, 0x94f4efa8, 0x91c3d373, 0x8ea4398a, 0x8b95c1e3, 0x88980e80, - 0x85aac367, 0x82cd8698, -}; - -/* - * Precomputed \Sum y^k { 1<=k<=n }. These are floor(true_value) to prevent - * over-estimates when re-combining. - */ -static const u32 runnable_avg_yN_sum[] = { - 0, 1002, 1982, 2941, 3880, 4798, 5697, 6576, 7437, 8279, 9103, - 9909,10698,11470,12226,12966,13690,14398,15091,15769,16433,17082, - 17718,18340,18949,19545,20128,20698,21256,21802,22336,22859,23371, -}; - -/* - * Approximate: - * val * y^n, where y^32 ~= 0.5 (~1 scheduling period) - */ -static __always_inline u64 decay_load(u64 val, u64 n) -{ - unsigned int local_n; - - if (!n) - return val; - else if (unlikely(n > LOAD_AVG_PERIOD * 63)) - return 0; - - /* after bounds checking we can collapse to 32-bit */ - local_n = n; - - /* - * As y^PERIOD = 1/2, we can combine - * y^n = 1/2^(n/PERIOD) * k^(n%PERIOD) - * With a look-up table which covers k^n (n= LOAD_AVG_PERIOD)) { - val >>= local_n / LOAD_AVG_PERIOD; - local_n %= LOAD_AVG_PERIOD; - } - - val *= runnable_avg_yN_inv[local_n]; - /* We don't use SRR here since we always want to round down. */ - return val >> 32; -} - -/* - * For updates fully spanning n periods, the contribution to runnable - * average will be: \Sum 1024*y^n - * - * We can compute this reasonably efficiently by combining: - * y^PERIOD = 1/2 with precomputed \Sum 1024*y^n {for n = LOAD_AVG_MAX_N)) - return LOAD_AVG_MAX; - - /* Compute \Sum k^n combining precomputed values for k^i, \Sum k^j */ - do { - contrib /= 2; /* y^LOAD_AVG_PERIOD = 1/2 */ - contrib += runnable_avg_yN_sum[LOAD_AVG_PERIOD]; - - n -= LOAD_AVG_PERIOD; - } while (n > LOAD_AVG_PERIOD); - - contrib = decay_load(contrib, n); - return contrib + runnable_avg_yN_sum[n]; -} - -static void add_to_scaled_stat(int cpu, struct sched_avg *sa, u64 delta); -static inline void decay_scaled_stat(struct sched_avg *sa, u64 periods); - -#ifdef CONFIG_SCHED_HMP - -/* Initial task load. Newly created tasks are assigned this load. */ -unsigned int __read_mostly sched_init_task_load_pelt; -unsigned int __read_mostly sched_init_task_load_windows; -unsigned int __read_mostly sysctl_sched_init_task_load_pct = 15; - -/* - * Keep these two below in sync. One is in unit of ns and the - * other in unit of us. - */ -unsigned int __read_mostly sysctl_sched_min_runtime = 0; /* 0 ms */ -u64 __read_mostly sched_min_runtime = 0; /* 0 ms */ - -static inline unsigned int task_load(struct task_struct *p) -{ - if (sched_use_pelt) - return p->se.avg.runnable_avg_sum_scaled; - - return p->ravg.demand; -} - -unsigned int max_task_load(void) -{ - if (sched_use_pelt) - return LOAD_AVG_MAX; - - return sched_ravg_window; -} - -/* Use this knob to turn on or off HMP-aware task placement logic */ -unsigned int __read_mostly sched_enable_hmp = 0; - -/* A cpu can no longer accomodate more tasks if: - * - * rq->nr_running > sysctl_sched_spill_nr_run || - * rq->hmp_stats.cumulative_runnable_avg > sched_spill_load - */ -unsigned int __read_mostly sysctl_sched_spill_nr_run = 10; - -/* - * Control whether or not individual CPU power consumption is used to - * guide task placement. - * This sysctl can be set to a default value using boot command line arguments. - */ -unsigned int __read_mostly sysctl_sched_enable_power_aware = 0; - -/* - * This specifies the maximum percent power difference between 2 - * CPUs for them to be considered identical in terms of their - * power characteristics (i.e. they are in the same power band). - */ -unsigned int __read_mostly sysctl_sched_powerband_limit_pct = 20; - -/* - * CPUs with load greater than the sched_spill_load_threshold are not - * eligible for task placement. When all CPUs in a cluster achieve a - * load higher than this level, tasks becomes eligible for inter - * cluster migration. - */ -unsigned int __read_mostly sched_spill_load; -unsigned int __read_mostly sysctl_sched_spill_load_pct = 100; - -/* - * Tasks whose bandwidth consumption on a cpu is less than - * sched_small_task are considered as small tasks. - */ -unsigned int __read_mostly sched_small_task; -unsigned int __read_mostly sysctl_sched_small_task_pct = 10; - -/* - * Tasks with demand >= sched_heavy_task will have their - * window-based demand added to the previous window's CPU - * time when they wake up, if they have slept for at least - * one full window. This feature is disabled when the tunable - * is set to 0 (the default). - */ -#ifdef CONFIG_SCHED_FREQ_INPUT -unsigned int __read_mostly sysctl_sched_heavy_task_pct; -unsigned int __read_mostly sched_heavy_task; -#endif - -/* - * Tasks whose bandwidth consumption on a cpu is more than - * sched_upmigrate are considered "big" tasks. Big tasks will be - * considered for "up" migration, i.e migrating to a cpu with better - * capacity. - */ -unsigned int __read_mostly sched_upmigrate; -unsigned int __read_mostly sysctl_sched_upmigrate_pct = 80; - -/* - * Big tasks, once migrated, will need to drop their bandwidth - * consumption to less than sched_downmigrate before they are "down" - * migrated. - */ -unsigned int __read_mostly sched_downmigrate; -unsigned int __read_mostly sysctl_sched_downmigrate_pct = 60; - -/* - * Tasks whose nice value is > sysctl_sched_upmigrate_min_nice are never - * considered as "big" tasks. - */ -static int __read_mostly sched_upmigrate_min_nice = 15; -int __read_mostly sysctl_sched_upmigrate_min_nice = 15; - -/* - * Scheduler boost is a mechanism to temporarily place tasks on CPUs - * with higher capacity than those where a task would have normally - * ended up with their load characteristics. Any entity enabling - * boost is responsible for disabling it as well. - */ -unsigned int sysctl_sched_boost; - -static inline int available_cpu_capacity(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - return rq->capacity; -} - -void set_hmp_defaults(void) -{ - sched_spill_load = - pct_to_real(sysctl_sched_spill_load_pct); - - sched_small_task = - pct_to_real(sysctl_sched_small_task_pct); - - sched_upmigrate = - pct_to_real(sysctl_sched_upmigrate_pct); - - sched_downmigrate = - pct_to_real(sysctl_sched_downmigrate_pct); - -#ifdef CONFIG_SCHED_FREQ_INPUT - sched_heavy_task = - pct_to_real(sysctl_sched_heavy_task_pct); -#endif - - sched_init_task_load_pelt = - div64_u64((u64)sysctl_sched_init_task_load_pct * - (u64)LOAD_AVG_MAX, 100); - - sched_init_task_load_windows = - div64_u64((u64)sysctl_sched_init_task_load_pct * - (u64)sched_ravg_window, 100); - - sched_upmigrate_min_nice = sysctl_sched_upmigrate_min_nice; -} - -u32 sched_get_init_task_load(struct task_struct *p) -{ - return p->init_load_pct; -} - -int sched_set_init_task_load(struct task_struct *p, int init_load_pct) -{ - if (init_load_pct < 0 || init_load_pct > 100) - return -EINVAL; - - p->init_load_pct = init_load_pct; - - return 0; -} - -int sched_set_cpu_prefer_idle(int cpu, int prefer_idle) -{ - struct rq *rq = cpu_rq(cpu); - - rq->prefer_idle = !!prefer_idle; - - return 0; -} - -int sched_get_cpu_prefer_idle(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - return rq->prefer_idle; -} - -int sched_set_cpu_mostly_idle_load(int cpu, int mostly_idle_pct) -{ - struct rq *rq = cpu_rq(cpu); - - if (mostly_idle_pct < 0 || mostly_idle_pct > 100) - return -EINVAL; - - rq->mostly_idle_load = pct_to_real(mostly_idle_pct); - - return 0; -} - -int sched_set_cpu_mostly_idle_freq(int cpu, unsigned int mostly_idle_freq) -{ - struct rq *rq = cpu_rq(cpu); - - if (mostly_idle_freq > rq->max_possible_freq) - return -EINVAL; - - rq->mostly_idle_freq = mostly_idle_freq; - - return 0; -} - -unsigned int sched_get_cpu_mostly_idle_freq(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - return rq->mostly_idle_freq; -} - -int sched_get_cpu_mostly_idle_load(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - int mostly_idle_pct; - - mostly_idle_pct = real_to_pct(rq->mostly_idle_load); - - return mostly_idle_pct; -} - -int sched_set_cpu_mostly_idle_nr_run(int cpu, int nr_run) -{ - struct rq *rq = cpu_rq(cpu); - - rq->mostly_idle_nr_run = nr_run; - - return 0; -} - -int sched_get_cpu_mostly_idle_nr_run(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - return rq->mostly_idle_nr_run; -} - -#ifdef CONFIG_CGROUP_SCHED - -static inline int upmigrate_discouraged(struct task_struct *p) -{ - return task_group(p)->upmigrate_discouraged; -} - -#else - -static inline int upmigrate_discouraged(struct task_struct *p) -{ - return 0; -} - -#endif - -/* Is a task "big" on its current cpu */ -static inline int is_big_task(struct task_struct *p) -{ - u64 load = task_load(p); - int nice = TASK_NICE(p); - - if (nice > sched_upmigrate_min_nice || upmigrate_discouraged(p)) - return 0; - - load = scale_load_to_cpu(load, task_cpu(p)); - - return load > sched_upmigrate; -} - -/* Is a task "small" on the minimum capacity CPU */ -static inline int is_small_task(struct task_struct *p) -{ - u64 load = task_load(p); - load *= (u64)max_load_scale_factor; - load /= 1024; - return load < sched_small_task; -} - -static inline u64 cpu_load(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - return scale_load_to_cpu(rq->hmp_stats.cumulative_runnable_avg, cpu); -} - -static inline u64 cpu_load_sync(int cpu, int sync) -{ - struct rq *rq = cpu_rq(cpu); - u64 load; - - load = rq->hmp_stats.cumulative_runnable_avg; - - /* - * If load is being checked in a sync wakeup environment, - * we may want to discount the load of the currently running - * task. - */ - if (sync && cpu == smp_processor_id()) { - if (load > rq->curr->ravg.demand) - load -= rq->curr->ravg.demand; - else - load = 0; - } - - return scale_load_to_cpu(load, cpu); -} - -static int -spill_threshold_crossed(u64 task_load, u64 cpu_load, struct rq *rq) -{ - u64 total_load = task_load + cpu_load; - - if (total_load > sched_spill_load || - (rq->nr_running + 1) > sysctl_sched_spill_nr_run) - return 1; - - return 0; -} - -int mostly_idle_cpu(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - return cpu_load(cpu) <= rq->mostly_idle_load - && rq->nr_running <= rq->mostly_idle_nr_run - && !sched_cpu_high_irqload(cpu); -} - -static int mostly_idle_cpu_sync(int cpu, u64 load, int sync) -{ - struct rq *rq = cpu_rq(cpu); - int nr_running; - - nr_running = rq->nr_running; - - /* - * Sync wakeups mean that the waker task will go to sleep - * soon so we should discount its load from this test. - */ - if (sync && cpu == smp_processor_id()) - nr_running--; - - return load <= rq->mostly_idle_load && - nr_running <= rq->mostly_idle_nr_run; -} - -static int boost_refcount; -static DEFINE_SPINLOCK(boost_lock); -static DEFINE_MUTEX(boost_mutex); - -static void boost_kick_cpus(void) -{ - int i; - - for_each_online_cpu(i) { - if (cpu_rq(i)->capacity != max_capacity) - boost_kick(i); - } -} - -int sched_boost(void) -{ - return boost_refcount > 0; -} - -int sched_set_boost(int enable) -{ - unsigned long flags; - int ret = 0; - int old_refcount; - - if (!sched_enable_hmp) - return -EINVAL; - - spin_lock_irqsave(&boost_lock, flags); - - old_refcount = boost_refcount; - - if (enable == 1) { - boost_refcount++; - } else if (!enable) { - if (boost_refcount >= 1) - boost_refcount--; - else - ret = -EINVAL; - } else { - ret = -EINVAL; - } - - if (!old_refcount && boost_refcount) - boost_kick_cpus(); - - trace_sched_set_boost(boost_refcount); - spin_unlock_irqrestore(&boost_lock, flags); - - return ret; -} - -int sched_boost_handler(struct ctl_table *table, int write, - void __user *buffer, size_t *lenp, - loff_t *ppos) -{ - int ret; - - mutex_lock(&boost_mutex); - if (!write) - sysctl_sched_boost = sched_boost(); - - ret = proc_dointvec(table, write, buffer, lenp, ppos); - if (ret || !write) - goto done; - - ret = (sysctl_sched_boost <= 1) ? - sched_set_boost(sysctl_sched_boost) : -EINVAL; - -done: - mutex_unlock(&boost_mutex); - return ret; -} - -static inline int is_cpu_throttling_imminent(int cpu); -#ifdef VENDOR_EDIT -void down_migrate_task(struct task_struct *p) -{ - if (p->ravg.mitigated) - return; - - if (sysctl_thermal_aware_scheduling) - p->ravg.mitigated = 1; -} -#endif - -/* - * Task will fit on a cpu if it's bandwidth consumption on that cpu - * will be less than sched_upmigrate. A big task that was previously - * "up" migrated will be considered fitting on "little" cpu if its - * bandwidth consumption on "little" cpu will be less than - * sched_downmigrate. This will help avoid frequenty migrations for - * tasks with load close to the upmigrate threshold - */ - -static int task_load_will_fit(struct task_struct *p, u64 tload, int cpu) -{ - struct rq *prev_rq = cpu_rq(task_cpu(p)); - struct rq *rq = cpu_rq(cpu); - int upmigrate, nice; -#ifdef VENDOR_EDIT - int thermal_mitigation = 0; - int prev_cpu_throttled = 0, cpu_throttled = 0, ta_enabled = 0; - int prev_cpu = task_cpu(p); -#endif - - if (rq->capacity == max_capacity) - return 1; - -#ifdef VENDOR_EDIT - if (p->ravg.mitigated) - return 1; -#endif - if (sched_boost()) { - if (rq->capacity > prev_rq->capacity) - return 1; - } else { - nice = TASK_NICE(p); - if (nice > sched_upmigrate_min_nice || upmigrate_discouraged(p)) - return 1; - - upmigrate = sched_upmigrate; -#ifdef VENDOR_EDIT - if (sysctl_thermal_aware_scheduling == SCHED_TA_THERMAL_ONLY) { - prev_cpu_throttled = is_cpu_throttling_imminent(prev_cpu); - cpu_throttled = is_cpu_throttling_imminent(cpu); - if (prev_cpu_throttled || cpu_throttled) - ta_enabled = 1; - } else if (sysctl_thermal_aware_scheduling == SCHED_TA_ALWAYS_ON) - ta_enabled = 1; - - if (ta_enabled && cpu != prev_cpu - && prev_rq->max_possible_capacity > rq->max_possible_capacity) { - thermal_mitigation = 1; - tload = scale_load_to_cpu(task_load(p), prev_cpu); - } -#endif - -#ifdef VENDOR_EDIT - if (prev_rq->capacity > rq->capacity || thermal_mitigation) -#else - if (prev_rq->capacity > rq->capacity) -#endif - upmigrate = sched_downmigrate; - - if (tload < upmigrate) { -#ifdef VENDOR_EDIT - if (thermal_mitigation) - down_migrate_task(p); -#endif - return 1; - } - } - - return 0; -} - -static int task_will_fit(struct task_struct *p, int cpu) -{ - u64 tload = scale_load_to_cpu(task_load(p), cpu); - return task_load_will_fit(p, tload, cpu); -} - -static int eligible_cpu(u64 task_load, u64 cpu_load, int cpu, int sync, unsigned task_mitigated) -{ - struct rq *rq = cpu_rq(cpu); - - if (sched_cpu_high_irqload(cpu)) - return 0; - - if (mostly_idle_cpu_sync(cpu, cpu_load, sync)) - return 1; - - if (rq->max_possible_capacity != max_possible_capacity) { -#ifdef VENDOR_EDIT - /* Current demand is still high, bail all small cpus out. - * select_best_cpu() will choose proper cpu to run */ - if (task_mitigated) - return 1; -#endif - return !spill_threshold_crossed(task_load, cpu_load, rq); - } - - return 0; -} - -struct cpu_pwr_stats __weak *get_cpu_pwr_stats(void) -{ - return NULL; -} - -int power_delta_exceeded(unsigned int cpu_cost, unsigned int base_cost) -{ - int delta, cost_limit; - - if (!base_cost || cpu_cost == base_cost) - return 0; - - delta = cpu_cost - base_cost; - cost_limit = div64_u64((u64)sysctl_sched_powerband_limit_pct * - (u64)base_cost, 100); - return abs(delta) > cost_limit; -} - -unsigned int power_cost_at_freq(int cpu, unsigned int freq) -{ - int i = 0; - struct cpu_pwr_stats *per_cpu_info = get_cpu_pwr_stats(); - struct cpu_pstate_pwr *costs; - - if (!per_cpu_info || !per_cpu_info[cpu].ptable || - !sysctl_sched_enable_power_aware) - /* When power aware scheduling is not in use, or CPU - * power data is not available, just use the CPU - * capacity as a rough stand-in for real CPU power - * numbers, assuming bigger CPUs are more power - * hungry. */ - return cpu_rq(cpu)->max_possible_capacity; - - if (!freq) - freq = min_max_freq; - - costs = per_cpu_info[cpu].ptable; - - while (costs[i].freq != 0) { - if (costs[i].freq >= freq || - costs[i+1].freq == 0) - return costs[i].power; - i++; - } - BUG(); -} - -/* Return the cost of running task p on CPU cpu. This function - * currently assumes that task p is the only task which will run on - * the CPU. */ -static unsigned int power_cost(u64 task_load, int cpu) -{ - unsigned int task_freq, cur_freq; - struct rq *rq = cpu_rq(cpu); - u64 demand; - - if (!sysctl_sched_enable_power_aware) - return rq->max_possible_capacity; - - /* calculate % of max freq needed */ - demand = task_load * 100; - demand = div64_u64(demand, max_task_load()); - - task_freq = demand * rq->max_possible_freq; - task_freq /= 100; /* khz needed */ + * If we raced with hotplug and there are no CPUs left in our mask + * the @ns structure is NULL'ed and task_numa_compare() will + * not find this node attractive. + * + * We'll either bail at !has_free_capacity, or we'll detect a huge + * imbalance and bail there. + */ + if (!cpus) + return; - cur_freq = rq->cur_freq; - task_freq = max(cur_freq, task_freq); + /* smt := ceil(cpus / capacity), assumes: 1 < smt_power < 2 */ + smt = DIV_ROUND_UP(SCHED_CAPACITY_SCALE * cpus, ns->compute_capacity); + capacity = cpus / smt; /* cores */ - return power_cost_at_freq(cpu, task_freq); + ns->task_capacity = min_t(unsigned, capacity, + DIV_ROUND_CLOSEST(ns->compute_capacity, SCHED_CAPACITY_SCALE)); + ns->has_free_capacity = (ns->nr_running < ns->task_capacity); } -static int best_small_task_cpu(struct task_struct *p, int sync) -{ - int best_busy_cpu = -1, fallback_cpu = -1; - int min_cstate_cpu = -1; - int min_cstate = INT_MAX; - int cpu_cost, min_cost = INT_MAX; - int i = task_cpu(p), prev_cpu; - int hmp_capable; - u64 tload, cpu_load, min_load = ULLONG_MAX; - cpumask_t temp; - cpumask_t search_cpu; - cpumask_t fb_search_cpu = CPU_MASK_NONE; - struct rq *rq; - - cpumask_and(&temp, &mpc_mask, cpu_possible_mask); - hmp_capable = !cpumask_full(&temp); - - cpumask_and(&search_cpu, tsk_cpus_allowed(p), cpu_online_mask); - if (unlikely(cpumask_empty(&search_cpu))) - return task_cpu(p); - if (unlikely(!cpumask_test_cpu(i, &search_cpu))) - i = cpumask_first(&search_cpu); - - do { - rq = cpu_rq(i); - - trace_sched_cpu_load(rq, idle_cpu(i), - mostly_idle_cpu_sync(i, - cpu_load_sync(i, sync), sync), - sched_irqload(i), - power_cost(scale_load_to_cpu(task_load(p), - i), i), - cpu_temp(i)); - - if (rq->max_possible_capacity == max_possible_capacity && - hmp_capable) { - cpumask_and(&fb_search_cpu, &search_cpu, - &rq->freq_domain_cpumask); - cpumask_andnot(&search_cpu, &search_cpu, - &rq->freq_domain_cpumask); - continue; - } - - cpumask_clear_cpu(i, &search_cpu); - - if (sched_cpu_high_irqload(i)) - continue; - - if (idle_cpu(i) && rq->cstate) { - if (rq->cstate < min_cstate) { - min_cstate_cpu = i; - min_cstate = rq->cstate; - } - continue; - } - - cpu_load = cpu_load_sync(i, sync); - if (mostly_idle_cpu_sync(i, cpu_load, sync)) - return i; - } while ((i = cpumask_first(&search_cpu)) < nr_cpu_ids); - - if (min_cstate_cpu != -1) - return min_cstate_cpu; - - cpumask_and(&search_cpu, tsk_cpus_allowed(p), cpu_online_mask); - cpumask_andnot(&search_cpu, &search_cpu, &fb_search_cpu); - for_each_cpu(i, &search_cpu) { - rq = cpu_rq(i); - prev_cpu = (i == task_cpu(p)); - - if (sched_cpu_high_irqload(i)) - continue; - - tload = scale_load_to_cpu(task_load(p), i); - cpu_load = cpu_load_sync(i, sync); - if (!spill_threshold_crossed(tload, cpu_load, rq)) { - if (cpu_load < min_load || - (prev_cpu && cpu_load == min_load)) { - min_load = cpu_load; - best_busy_cpu = i; - } - } - } +struct task_numa_env { + struct task_struct *p; - if (best_busy_cpu != -1) - return best_busy_cpu; + int src_cpu, src_nid; + int dst_cpu, dst_nid; - for_each_cpu(i, &fb_search_cpu) { - rq = cpu_rq(i); - prev_cpu = (i == task_cpu(p)); - - tload = scale_load_to_cpu(task_load(p), i); - cpu_cost = power_cost(tload, i); - if (cpu_cost < min_cost || - (prev_cpu && cpu_cost == min_cost)) { - fallback_cpu = i; - min_cost = cpu_cost; - } - } + struct numa_stats src_stats, dst_stats; - return fallback_cpu; -} + int imbalance_pct; -#define UP_MIGRATION 1 -#define DOWN_MIGRATION 2 -#define EA_MIGRATION 3 -#define IRQLOAD_MIGRATION 4 + struct task_struct *best_task; + long best_imp; + int best_cpu; +}; -static int skip_freq_domain(struct rq *task_rq, struct rq *rq, int reason) +static void task_numa_assign(struct task_numa_env *env, + struct task_struct *p, long imp) { - int skip; - - if (!reason) - return 0; - - switch (reason) { - case UP_MIGRATION: - skip = rq->capacity <= task_rq->capacity; - break; - - case DOWN_MIGRATION: - skip = rq->capacity >= task_rq->capacity; - break; - - case EA_MIGRATION: -#ifdef VENDOR_EDIT - if (!sysctl_thermal_aware_scheduling) -#endif - skip = rq->capacity != task_rq->capacity; - break; - - case IRQLOAD_MIGRATION: - /* Purposely fall through */ - - default: - return 0; - } + if (env->best_task) + put_task_struct(env->best_task); + if (p) + get_task_struct(p); - return skip; + env->best_task = p; + env->best_imp = imp; + env->best_cpu = env->dst_cpu; } -static int skip_cpu(struct rq *task_rq, struct rq *rq, int cpu, - u64 task_load, int reason) +static bool load_too_imbalanced(long src_load, long dst_load, + struct task_numa_env *env) { - int skip; - - if (!reason) - return 0; - - if (is_reserved(cpu)) - return 1; - - switch (reason) { - case EA_MIGRATION: - skip = power_cost(task_load, cpu) > - power_cost(task_load, cpu_of(task_rq)); - break; - - case IRQLOAD_MIGRATION: - /* Purposely fall through */ - - default: - skip = (rq == task_rq); - } - - return skip; -} + long imb, old_imb; + long orig_src_load, orig_dst_load; + long src_capacity, dst_capacity; -/* - * Select a single cpu in cluster as target for packing, iff cluster frequency - * is less than a threshold level - */ -static int select_packing_target(struct task_struct *p, int best_cpu) -{ - struct rq *rq = cpu_rq(best_cpu); - struct cpumask search_cpus; - int i; - int min_cost = INT_MAX; - int target = best_cpu; + /* + * The load is corrected for the CPU capacity available on each node. + * + * src_load dst_load + * ------------ vs --------- + * src_capacity dst_capacity + */ + src_capacity = env->src_stats.compute_capacity; + dst_capacity = env->dst_stats.compute_capacity; - if (rq->cur_freq >= rq->mostly_idle_freq) - return best_cpu; + /* We care about the slope of the imbalance, not the direction. */ + if (dst_load < src_load) + swap(dst_load, src_load); - /* Don't pack if current freq is low because of throttling */ - if (rq->max_freq <= rq->mostly_idle_freq) - return best_cpu; + /* Is the difference below the threshold? */ + imb = dst_load * src_capacity * 100 - + src_load * dst_capacity * env->imbalance_pct; + if (imb <= 0) + return false; - cpumask_and(&search_cpus, tsk_cpus_allowed(p), cpu_online_mask); - cpumask_and(&search_cpus, &search_cpus, &rq->freq_domain_cpumask); + /* + * The imbalance is above the allowed threshold. + * Compare it with the old imbalance. + */ + orig_src_load = env->src_stats.load; + orig_dst_load = env->dst_stats.load; - /* Pick the first lowest power cpu as target */ - for_each_cpu(i, &search_cpus) { - int cost = power_cost(scale_load_to_cpu(task_load(p), i), i); + if (orig_dst_load < orig_src_load) + swap(orig_dst_load, orig_src_load); - if (cost < min_cost && !sched_cpu_high_irqload(i)) { - target = i; - min_cost = cost; - } - } + old_imb = orig_dst_load * src_capacity * 100 - + orig_src_load * dst_capacity * env->imbalance_pct; - return target; + /* Would this change make things worse? */ + return (imb > old_imb); } /* - * Should task be woken to any available idle cpu? - * - * Waking tasks to idle cpu has mixed implications on both performance and - * power. In many cases, scheduler can't estimate correctly impact of using idle - * cpus on either performance or power. PF_WAKE_UP_IDLE allows external kernel - * module to pass a strong hint to scheduler that the task in question should be - * woken to idle cpu, generally to improve performance. - */ -static inline int wake_to_idle(struct task_struct *p) -{ - return (current->flags & PF_WAKE_UP_IDLE) || - (p->flags & PF_WAKE_UP_IDLE); -} - -/* return cheapest cpu that can fit this task */ -static int select_best_cpu(struct task_struct *p, int target, int reason, - int sync) -{ - int i, j, prev_cpu, best_cpu = -1; - int fallback_idle_cpu = -1, min_cstate_cpu = -1; - int cpu_cost, min_cost = INT_MAX; - int min_idle_cost = INT_MAX, min_busy_cost = INT_MAX; - u64 tload, cpu_load; - u64 min_load = ULLONG_MAX, min_fallback_load = ULLONG_MAX; - int small_task = is_small_task(p); - int boost = sched_boost(); - int cstate, min_cstate = INT_MAX; - int prefer_idle = -1; - int prefer_idle_override = 0; - cpumask_t search_cpus; - struct rq *trq; - - if (reason) { - prefer_idle = 1; - prefer_idle_override = 1; - } - - if (wake_to_idle(p)) { - prefer_idle = 1; - prefer_idle_override = 1; - small_task = 0; - /* - * If wake to idle and sync are both set prefer wake to idle - * since sync is a weak hint that might not always be correct. - */ - sync = 0; - } - - if (small_task && !boost) { - best_cpu = best_small_task_cpu(p, sync); - prefer_idle = 0; /* For sched_task_load tracepoint */ - goto done; - } - - trq = task_rq(p); - cpumask_and(&search_cpus, tsk_cpus_allowed(p), cpu_online_mask); - for_each_cpu(i, &search_cpus) { - struct rq *rq = cpu_rq(i); - - trace_sched_cpu_load(cpu_rq(i), idle_cpu(i), - mostly_idle_cpu_sync(i, - cpu_load_sync(i, sync), sync), - sched_irqload(i), - power_cost(scale_load_to_cpu(task_load(p), - i), i), - cpu_temp(i)); - - if (skip_freq_domain(trq, rq, reason)) { - cpumask_andnot(&search_cpus, &search_cpus, - &rq->freq_domain_cpumask); - continue; - } - - tload = scale_load_to_cpu(task_load(p), i); - if (skip_cpu(trq, rq, i, tload, reason)) - continue; - - prev_cpu = (i == task_cpu(p)); - - /* - * The least-loaded mostly-idle CPU where the task - * won't fit is our fallback if we can't find a CPU - * where the task will fit. - */ - if (!task_load_will_fit(p, tload, i)) { - for_each_cpu_and(j, &search_cpus, - &rq->freq_domain_cpumask) { - cpu_load = cpu_load_sync(j, sync); - if (mostly_idle_cpu_sync(j, cpu_load, sync) && - !sched_cpu_high_irqload(j)) { - if (cpu_load < min_fallback_load || - (cpu_load == min_fallback_load && - j == task_cpu(p))) { - min_fallback_load = cpu_load; - fallback_idle_cpu = j; - } - } - } - cpumask_andnot(&search_cpus, &search_cpus, - &rq->freq_domain_cpumask); - continue; - } + * This checks if the overall compute and NUMA accesses of the system would + * be improved if the source tasks was migrated to the target dst_cpu taking + * into account that it might be best if task running on the dst_cpu should + * be exchanged with the source task + */ +static void task_numa_compare(struct task_numa_env *env, + long taskimp, long groupimp) +{ + struct rq *src_rq = cpu_rq(env->src_cpu); + struct rq *dst_rq = cpu_rq(env->dst_cpu); + struct task_struct *cur; + long src_load, dst_load; + long load; + long imp = env->p->numa_group ? groupimp : taskimp; + long moveimp = imp; - /* Set prefer_idle based on the cpu where task will first fit */ - if (prefer_idle == -1) - prefer_idle = cpu_rq(i)->prefer_idle; + rcu_read_lock(); - cpu_load = cpu_load_sync(i, sync); - if (!eligible_cpu(tload, cpu_load, i, sync, p->ravg.mitigated)) - continue; + raw_spin_lock_irq(&dst_rq->lock); + cur = dst_rq->curr; + /* + * No need to move the exiting task, and this ensures that ->curr + * wasn't reaped and thus get_task_struct() in task_numa_assign() + * is safe under RCU read lock. + * Note that rcu_read_lock() itself can't protect from the final + * put_task_struct() after the last schedule(). + */ + if ((cur->flags & PF_EXITING) || is_idle_task(cur)) + cur = NULL; + raw_spin_unlock_irq(&dst_rq->lock); - /* - * The task will fit on this CPU, and the CPU is either - * mostly_idle or not max capacity and can fit it under - * spill. - */ + /* + * Because we have preemption enabled we can get migrated around and + * end try selecting ourselves (current == env->p) as a swap candidate. + */ + if (cur == env->p) + goto unlock; - cpu_cost = power_cost(tload, i); + /* + * "imp" is the fault differential for the source task between the + * source and destination node. Calculate the total differential for + * the source task and potential destination task. The more negative + * the value is, the more rmeote accesses that would be expected to + * be incurred if the tasks were swapped. + */ + if (cur) { + /* Skip this swap candidate if cannot move to the source cpu */ + if (!cpumask_test_cpu(env->src_cpu, tsk_cpus_allowed(cur))) + goto unlock; /* - * If the task fits in a CPU in a lower power band, that - * overrides load and C-state. + * If dst and source tasks are in the same NUMA group, or not + * in any group then look only at task weights. */ - if (power_delta_exceeded(cpu_cost, min_cost)) { - if (cpu_cost > min_cost) - continue; - - min_cost = cpu_cost; - min_load = ULLONG_MAX; - min_cstate = INT_MAX; - min_cstate_cpu = -1; - best_cpu = -1; - if (!prefer_idle_override) - prefer_idle = cpu_rq(i)->prefer_idle; + if (cur->numa_group == env->p->numa_group) { + imp = taskimp + task_weight(cur, env->src_nid) - + task_weight(cur, env->dst_nid); + /* + * Add some hysteresis to prevent swapping the + * tasks within a group over tiny differences. + */ + if (cur->numa_group) + imp -= imp/16; + } else { + /* + * Compare the group weights. If a task is all by + * itself (not part of a group), use the task weight + * instead. + */ + if (cur->numa_group) + imp += group_weight(cur, env->src_nid) - + group_weight(cur, env->dst_nid); + else + imp += task_weight(cur, env->src_nid) - + task_weight(cur, env->dst_nid); } + } - /* - * Partition CPUs based on whether they are completely idle - * or not. For completely idle CPUs we choose the one in - * the lowest C-state and then break ties with power cost. - * - * For sync wakeups we only consider the waker CPU as idle if - * prefer_idle is set. Otherwise if prefer_idle is unset sync - * wakeups will get biased away from the waker CPU. - */ - if (idle_cpu(i) || (sync && i == smp_processor_id() - && prefer_idle && cpu_rq(i)->nr_running == 1)) { - cstate = cpu_rq(i)->cstate; - - if (cstate > min_cstate) - continue; - - if (cstate < min_cstate) { - min_idle_cost = cpu_cost; - min_cstate = cstate; - min_cstate_cpu = i; - continue; - } + if (imp <= env->best_imp && moveimp <= env->best_imp) + goto unlock; - if (cpu_cost < min_idle_cost || - (prev_cpu && cpu_cost == min_idle_cost)) { - min_idle_cost = cpu_cost; - min_cstate_cpu = i; - } + if (!cur) { + /* Is there capacity at our destination? */ + if (env->src_stats.nr_running <= env->src_stats.task_capacity && + !env->dst_stats.has_free_capacity) + goto unlock; - continue; - } + goto balance; + } - /* - * For CPUs that are not completely idle, pick one with the - * lowest load and break ties with power cost - */ - if (cpu_load > min_load) - continue; + /* Balance doesn't matter much if we're running a task per cpu */ + if (imp > env->best_imp && src_rq->nr_running == 1 && + dst_rq->nr_running == 1) + goto assign; - if (cpu_load < min_load) { - min_load = cpu_load; - min_busy_cost = cpu_cost; - best_cpu = i; - continue; - } + /* + * In the overloaded case, try and keep the load balanced. + */ +balance: + load = task_h_load(env->p); + dst_load = env->dst_stats.load + load; + src_load = env->src_stats.load - load; + if (moveimp > imp && moveimp > env->best_imp) { /* - * The load is equal to the previous selected CPU. - * This is rare but when it does happen opt for the - * more power efficient CPU option. + * If the improvement from just moving env->p direction is + * better than swapping tasks around, check if a move is + * possible. Store a slightly smaller score than moveimp, + * so an actually idle CPU will win. */ - if (cpu_cost < min_busy_cost || - (prev_cpu && cpu_cost == min_busy_cost)) { - min_busy_cost = cpu_cost; - best_cpu = i; + if (!load_too_imbalanced(src_load, dst_load, env)) { + imp = moveimp - 1; + cur = NULL; + goto assign; } } - /* - * Don't need to check !sched_cpu_high_irqload(best_cpu) because - * best_cpu cannot have high irq load. - */ - if (min_cstate_cpu >= 0 && (prefer_idle > 0 || best_cpu < 0 || - !mostly_idle_cpu_sync(best_cpu, min_load, sync))) - best_cpu = min_cstate_cpu; -done: - if (best_cpu < 0) { - if (unlikely(fallback_idle_cpu < 0)) - /* - * For the lack of a better choice just use - * prev_cpu. We may just benefit from having - * a hot cache. - */ - best_cpu = task_cpu(p); - else - best_cpu = fallback_idle_cpu; + if (imp <= env->best_imp) + goto unlock; + + if (cur) { + load = task_h_load(cur); + dst_load -= load; + src_load += load; } - if (cpu_rq(best_cpu)->mostly_idle_freq && !prefer_idle_override) - best_cpu = select_packing_target(p, best_cpu); + if (load_too_imbalanced(src_load, dst_load, env)) + goto unlock; /* - * prefer_idle is initialized towards middle of function. Leave this - * tracepoint towards end to capture prefer_idle flag used for this - * instance of wakeup. + * One idle CPU per node is evaluated for a task numa move. + * Call select_idle_sibling to maybe find a better one. */ - trace_sched_task_load(p, small_task, boost, reason, sync, prefer_idle); - - return best_cpu; -} - -static void -inc_nr_big_small_task(struct hmp_sched_stats *stats, struct task_struct *p) -{ - if (!sched_enable_hmp || sched_disable_window_stats) - return; - - if (is_big_task(p)) - stats->nr_big_tasks++; - else if (is_small_task(p)) - stats->nr_small_tasks++; -} - -static void -dec_nr_big_small_task(struct hmp_sched_stats *stats, struct task_struct *p) -{ - if (!sched_enable_hmp || sched_disable_window_stats) - return; - - if (is_big_task(p)) - stats->nr_big_tasks--; - else if (is_small_task(p)) - stats->nr_small_tasks--; + if (!cur) + env->dst_cpu = select_idle_sibling(env->p, env->dst_cpu); - BUG_ON(stats->nr_big_tasks < 0 || stats->nr_small_tasks < 0); +assign: + task_numa_assign(env, cur, imp); +unlock: + rcu_read_unlock(); } -static void -inc_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) +static void task_numa_find_cpu(struct task_numa_env *env, + long taskimp, long groupimp) { - inc_nr_big_small_task(&rq->hmp_stats, p); - if (change_cra) - inc_cumulative_runnable_avg(&rq->hmp_stats, p); -} + int cpu; -static void -dec_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) -{ - dec_nr_big_small_task(&rq->hmp_stats, p); - if (change_cra) - dec_cumulative_runnable_avg(&rq->hmp_stats, p); -} + for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) { + /* Skip this CPU if the source task cannot migrate */ + if (!cpumask_test_cpu(cpu, tsk_cpus_allowed(env->p))) + continue; -static void reset_hmp_stats(struct hmp_sched_stats *stats, int reset_cra) -{ - stats->nr_big_tasks = stats->nr_small_tasks = 0; - if (reset_cra) - stats->cumulative_runnable_avg = 0; + env->dst_cpu = cpu; + task_numa_compare(env, taskimp, groupimp); + } } - -#ifdef CONFIG_CFS_BANDWIDTH - -static inline struct task_group *next_task_group(struct task_group *tg) +static int task_numa_migrate(struct task_struct *p) { - tg = list_entry_rcu(tg->list.next, typeof(struct task_group), list); + struct task_numa_env env = { + .p = p, - return (&tg->list == &task_groups) ? NULL : tg; -} + .src_cpu = task_cpu(p), + .src_nid = task_node(p), -/* Iterate over all cfs_rq in a cpu */ -#define for_each_cfs_rq(cfs_rq, tg, cpu) \ - for (tg = container_of(&task_groups, struct task_group, list); \ - ((tg = next_task_group(tg)) && (cfs_rq = tg->cfs_rq[cpu]));) + .imbalance_pct = 112, -static void reset_cfs_rq_hmp_stats(int cpu, int reset_cra) -{ - struct task_group *tg; - struct cfs_rq *cfs_rq; + .best_task = NULL, + .best_imp = 0, + .best_cpu = -1 + }; + struct sched_domain *sd; + unsigned long taskweight, groupweight; + int nid, ret; + long taskimp, groupimp; + /* + * Pick the lowest SD_NUMA domain, as that would have the smallest + * imbalance and would be the first to start moving tasks about. + * + * And we want to avoid any moving of tasks about, as that would create + * random movement of tasks -- counter the numa conditions we're trying + * to satisfy here. + */ rcu_read_lock(); - - for_each_cfs_rq(cfs_rq, tg, cpu) - reset_hmp_stats(&cfs_rq->hmp_stats, reset_cra); - + sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu)); + if (sd) + env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; rcu_read_unlock(); -} - -#else /* CONFIG_CFS_BANDWIDTH */ - -static inline void reset_cfs_rq_hmp_stats(int cpu, int reset_cra) { } - -#endif /* CONFIG_CFS_BANDWIDTH */ - -/* - * Return total number of tasks "eligible" to run on highest capacity cpu - * - * This is simply nr_big_tasks for cpus which are not of max_capacity and - * (nr_running - nr_small_tasks) for cpus of max_capacity - */ -unsigned int nr_eligible_big_tasks(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - int nr_big = rq->hmp_stats.nr_big_tasks; - int nr = rq->nr_running; - int nr_small = rq->hmp_stats.nr_small_tasks; - if (rq->max_possible_capacity != max_possible_capacity) - return nr_big; - - /* Consider all (except small) tasks on max_capacity cpu as big tasks */ - nr_big = nr - nr_small; - if (nr_big < 0) - nr_big = 0; - - return nr_big; -} + /* + * Cpusets can break the scheduler domain tree into smaller + * balance domains, some of which do not cross NUMA boundaries. + * Tasks that are "trapped" in such domains cannot be migrated + * elsewhere, so there is no point in (re)trying. + */ + if (unlikely(!sd)) { + p->numa_preferred_nid = task_node(p); + return -EINVAL; + } -/* - * reset_cpu_hmp_stats - reset HMP stats for a cpu - * nr_big_tasks, nr_small_tasks - * cumulative_runnable_avg (iff reset_cra is true) - */ -void reset_cpu_hmp_stats(int cpu, int reset_cra) -{ - reset_cfs_rq_hmp_stats(cpu, reset_cra); - reset_hmp_stats(&cpu_rq(cpu)->hmp_stats, reset_cra); -} + taskweight = task_weight(p, env.src_nid); + groupweight = group_weight(p, env.src_nid); + update_numa_stats(&env.src_stats, env.src_nid); + env.dst_nid = p->numa_preferred_nid; + taskimp = task_weight(p, env.dst_nid) - taskweight; + groupimp = group_weight(p, env.dst_nid) - groupweight; + update_numa_stats(&env.dst_stats, env.dst_nid); -#ifdef CONFIG_CFS_BANDWIDTH + /* Try to find a spot on the preferred nid. */ + task_numa_find_cpu(&env, taskimp, groupimp); -static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); + /* No space available on the preferred nid. Look elsewhere. */ + if (env.best_cpu == -1) { + for_each_online_node(nid) { + if (nid == env.src_nid || nid == p->numa_preferred_nid) + continue; -static void inc_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra); -static void dec_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra); + /* Only consider nodes where both task and groups benefit */ + taskimp = task_weight(p, nid) - taskweight; + groupimp = group_weight(p, nid) - groupweight; + if (taskimp < 0 && groupimp < 0) + continue; -/* Add task's contribution to a cpu' HMP statistics */ -static void -_inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p, int change_cra) -{ - struct cfs_rq *cfs_rq; - struct sched_entity *se = &p->se; + env.dst_nid = nid; + update_numa_stats(&env.dst_stats, env.dst_nid); + task_numa_find_cpu(&env, taskimp, groupimp); + } + } /* - * Although below check is not strictly required (as - * inc/dec_nr_big_small_task and inc/dec_cumulative_runnable_avg called - * from inc_cfs_rq_hmp_stats() have similar checks), we gain a bit on - * efficiency by short-circuiting for_each_sched_entity() loop when - * !sched_enable_hmp || sched_disable_window_stats + * If the task is part of a workload that spans multiple NUMA nodes, + * and is migrating into one of the workload's active nodes, remember + * this node as the task's preferred numa node, so the workload can + * settle down. + * A task that migrated to a second choice node will be better off + * trying for a better one later. Do not set the preferred node here. */ - if (!sched_enable_hmp || sched_disable_window_stats) - return; + if (p->numa_group) { + if (env.best_cpu == -1) + nid = env.src_nid; + else + nid = env.dst_nid; - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - inc_cfs_rq_hmp_stats(cfs_rq, p, change_cra); - if (cfs_rq_throttled(cfs_rq)) - break; + if (node_isset(nid, p->numa_group->active_nodes)) + sched_setnuma(p, env.dst_nid); } - /* Update rq->hmp_stats only if we didn't find any throttled cfs_rq */ - if (!se) - inc_rq_hmp_stats(rq, p, change_cra); -} - -/* Remove task's contribution from a cpu' HMP statistics */ -static void -_dec_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p, int change_cra) -{ - struct cfs_rq *cfs_rq; - struct sched_entity *se = &p->se; + /* No better CPU than the current one was found. */ + if (env.best_cpu == -1) + return -EAGAIN; - /* See comment on efficiency in _inc_hmp_sched_stats_fair */ - if (!sched_enable_hmp || sched_disable_window_stats) - return; + /* + * Reset the scan period if the task is being rescheduled on an + * alternative node to recheck if the tasks is now properly placed. + */ + p->numa_scan_period = task_scan_min(p); - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - dec_cfs_rq_hmp_stats(cfs_rq, p, change_cra); - if (cfs_rq_throttled(cfs_rq)) - break; + if (env.best_task == NULL) { + ret = migrate_task_to(p, env.best_cpu); + if (ret != 0) + trace_sched_stick_numa(p, env.src_cpu, env.best_cpu); + return ret; } - /* Update rq->hmp_stats only if we didn't find any throttled cfs_rq */ - if (!se) - dec_rq_hmp_stats(rq, p, change_cra); -} - -static void inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p) -{ - _inc_hmp_sched_stats_fair(rq, p, 1); -} - -static void dec_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p) -{ - _dec_hmp_sched_stats_fair(rq, p, 1); -} - -static int task_will_be_throttled(struct task_struct *p); - -#else /* CONFIG_CFS_BANDWIDTH */ - -static void -inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p) -{ - inc_nr_big_small_task(&rq->hmp_stats, p); - inc_cumulative_runnable_avg(&rq->hmp_stats, p); -} - -static void -dec_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p) -{ - dec_nr_big_small_task(&rq->hmp_stats, p); - dec_cumulative_runnable_avg(&rq->hmp_stats, p); -} - -static inline int task_will_be_throttled(struct task_struct *p) -{ - return 0; + ret = migrate_swap(p, env.best_task); + if (ret != 0) + trace_sched_stick_numa(p, env.src_cpu, task_cpu(env.best_task)); + put_task_struct(env.best_task); + return ret; } -static void -_inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p, int change_cra) +/* Attempt to migrate a task to a CPU on the preferred node. */ +static void numa_migrate_preferred(struct task_struct *p) { - inc_nr_big_small_task(&rq->hmp_stats, p); -} + unsigned long interval = HZ; -#endif /* CONFIG_CFS_BANDWIDTH */ + /* This task has no NUMA fault statistics yet */ + if (unlikely(p->numa_preferred_nid == -1 || !p->numa_faults_memory)) + return; -/* - * Walk runqueue of cpu and re-initialize 'nr_big_tasks' and 'nr_small_tasks' - * counters. - */ -void fixup_nr_big_small_task(int cpu, int reset_stats) -{ - struct rq *rq = cpu_rq(cpu); - struct task_struct *p; + /* Periodically retry migrating the task to the preferred node */ + interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); + p->numa_migrate_retry = jiffies + interval; - /* fixup_nr_big_small_task() is called from two functions. In one of - * them stats are already reset, don't waste time resetting them again - */ - if (reset_stats) { - /* Do not reset cumulative_runnable_avg */ - reset_cpu_hmp_stats(cpu, 0); - } + /* Success if task is already running on preferred CPU */ + if (task_node(p) == p->numa_preferred_nid) + return; - list_for_each_entry(p, &rq->cfs_tasks, se.group_node) - _inc_hmp_sched_stats_fair(rq, p, 0); + /* Otherwise, try migrate to a CPU on the preferred node */ + task_numa_migrate(p); } -/* Disable interrupts and grab runqueue lock of all cpus listed in @cpus */ -void pre_big_small_task_count_change(const struct cpumask *cpus) +/* + * Find the nodes on which the workload is actively running. We do this by + * tracking the nodes from which NUMA hinting faults are triggered. This can + * be different from the set of nodes where the workload's memory is currently + * located. + * + * The bitmask is used to make smarter decisions on when to do NUMA page + * migrations, To prevent flip-flopping, and excessive page migrations, nodes + * are added when they cause over 6/16 of the maximum number of faults, but + * only removed when they drop below 3/16. + */ +static void update_numa_active_node_mask(struct numa_group *numa_group) { - int i; + unsigned long faults, max_faults = 0; + int nid; - local_irq_disable(); + for_each_online_node(nid) { + faults = group_faults_cpu(numa_group, nid); + if (faults > max_faults) + max_faults = faults; + } - for_each_cpu(i, cpus) - raw_spin_lock(&cpu_rq(i)->lock); + for_each_online_node(nid) { + faults = group_faults_cpu(numa_group, nid); + if (!node_isset(nid, numa_group->active_nodes)) { + if (faults > max_faults * 6 / 16) + node_set(nid, numa_group->active_nodes); + } else if (faults < max_faults * 3 / 16) + node_clear(nid, numa_group->active_nodes); + } } /* - * Reinitialize 'nr_big_tasks' and 'nr_small_tasks' counters on all affected - * cpus + * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS + * increments. The more local the fault statistics are, the higher the scan + * period will be for the next scan window. If local/(local+remote) ratio is + * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) + * the scan period will decrease. Aim for 70% local accesses. */ -void post_big_small_task_count_change(const struct cpumask *cpus) +#define NUMA_PERIOD_SLOTS 10 +#define NUMA_PERIOD_THRESHOLD 7 + +/* + * Increase the scan period (slow down scanning) if the majority of + * our memory is already on our local node, or if the majority of + * the page accesses are shared with other processes. + * Otherwise, decrease the scan period. + */ +static void update_task_scan_period(struct task_struct *p, + unsigned long shared, unsigned long private) { - int i; + unsigned int period_slot; + int ratio; + int diff; - /* Assumes local_irq_disable() keeps online cpumap stable */ - for_each_cpu(i, cpus) - fixup_nr_big_small_task(i, 1); + unsigned long remote = p->numa_faults_locality[0]; + unsigned long local = p->numa_faults_locality[1]; - for_each_cpu(i, cpus) - raw_spin_unlock(&cpu_rq(i)->lock); + /* + * If there were no record hinting faults then either the task is + * completely idle or all activity is areas that are not of interest + * to automatic numa balancing. Scan slower + */ + if (local + shared == 0) { + p->numa_scan_period = min(p->numa_scan_period_max, + p->numa_scan_period << 1); - local_irq_enable(); -} + p->mm->numa_next_scan = jiffies + + msecs_to_jiffies(p->numa_scan_period); -DEFINE_MUTEX(policy_mutex); + return; + } -#ifdef CONFIG_SCHED_FREQ_INPUT -static inline int invalid_value_freq_input(unsigned int *data) -{ - if (data == &sysctl_sched_migration_fixup) - return !(*data == 0 || *data == 1); + /* + * Prepare to scale scan period relative to the current period. + * == NUMA_PERIOD_THRESHOLD scan period stays the same + * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) + * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) + */ + period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); + ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); + if (ratio >= NUMA_PERIOD_THRESHOLD) { + int slot = ratio - NUMA_PERIOD_THRESHOLD; + if (!slot) + slot = 1; + diff = slot * period_slot; + } else { + diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; - if (data == &sysctl_sched_freq_account_wait_time) - return !(*data == 0 || *data == 1); + /* + * Scale scan rate increases based on sharing. There is an + * inverse relationship between the degree of sharing and + * the adjustment made to the scanning period. Broadly + * speaking the intent is that there is little point + * scanning faster if shared accesses dominate as it may + * simply bounce migrations uselessly + */ + ratio = DIV_ROUND_UP(private * NUMA_PERIOD_SLOTS, (private + shared + 1)); + diff = (diff * ratio) / NUMA_PERIOD_SLOTS; + } - return 0; + p->numa_scan_period = clamp(p->numa_scan_period + diff, + task_scan_min(p), task_scan_max(p)); + memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); } -#else -static inline int invalid_value_freq_input(unsigned int *data) -{ - return 0; + +/* + * Get the fraction of time the task has been running since the last + * NUMA placement cycle. The scheduler keeps similar statistics, but + * decays those on a 32ms period, which is orders of magnitude off + * from the dozens-of-seconds NUMA balancing period. Use the scheduler + * stats only if the task is so new there are no NUMA statistics yet. + */ +static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) +{ + u64 runtime, delta, now; + /* Use the start of this time slice to avoid calculations. */ + now = p->se.exec_start; + runtime = p->se.sum_exec_runtime; + + if (p->last_task_numa_placement) { + delta = runtime - p->last_sum_exec_runtime; + *period = now - p->last_task_numa_placement; + } else { + delta = p->se.avg.runnable_avg_sum; + *period = p->se.avg.avg_period; + } + + p->last_sum_exec_runtime = runtime; + p->last_task_numa_placement = now; + + return delta; } -#endif -static inline int invalid_value(unsigned int *data) +static void task_numa_placement(struct task_struct *p) { - unsigned int val = *data; + int seq, nid, max_nid = -1, max_group_nid = -1; + unsigned long max_faults = 0, max_group_faults = 0; + unsigned long fault_types[2] = { 0, 0 }; + unsigned long total_faults; + u64 runtime, period; + spinlock_t *group_lock = NULL; - if (data == &sysctl_sched_ravg_hist_size) - return (val < 2 || val > RAVG_HIST_SIZE_MAX); + seq = ACCESS_ONCE(p->mm->numa_scan_seq); + if (p->numa_scan_seq == seq) + return; + p->numa_scan_seq = seq; + p->numa_scan_period_max = task_scan_max(p); - if (data == &sysctl_sched_window_stats_policy) - return val >= WINDOW_STATS_INVALID_POLICY; + total_faults = p->numa_faults_locality[0] + + p->numa_faults_locality[1]; + runtime = numa_get_avg_runtime(p, &period); - if (data == &sysctl_sched_account_wait_time) - return !(val == 0 || val == 1); + /* If the task is part of a group prevent parallel updates to group stats */ + if (p->numa_group) { + group_lock = &p->numa_group->lock; + spin_lock_irq(group_lock); + } - return invalid_value_freq_input(data); -} + /* Find the node with the highest number of faults */ + for_each_online_node(nid) { + unsigned long faults = 0, group_faults = 0; + int priv, i; -/* - * Handle "atomic" update of sysctl_sched_window_stats_policy, - * sysctl_sched_ravg_hist_size, sysctl_sched_account_wait_time and - * sched_freq_legacy_mode variables. - */ -int sched_window_update_handler(struct ctl_table *table, int write, - void __user *buffer, size_t *lenp, - loff_t *ppos) -{ - int ret; - unsigned int *data = (unsigned int *)table->data; - unsigned int old_val; + for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { + long diff, f_diff, f_weight; - if (!sched_enable_hmp) - return -EINVAL; + i = task_faults_idx(nid, priv); + + /* Decay existing window, copy faults since last scan */ + diff = p->numa_faults_buffer_memory[i] - p->numa_faults_memory[i] / 2; + fault_types[priv] += p->numa_faults_buffer_memory[i]; + p->numa_faults_buffer_memory[i] = 0; + + /* + * Normalize the faults_from, so all tasks in a group + * count according to CPU use, instead of by the raw + * number of faults. Tasks with little runtime have + * little over-all impact on throughput, and thus their + * faults are less important. + */ + f_weight = div64_u64(runtime << 16, period + 1); + f_weight = (f_weight * p->numa_faults_buffer_cpu[i]) / + (total_faults + 1); + f_diff = f_weight - p->numa_faults_cpu[i] / 2; + p->numa_faults_buffer_cpu[i] = 0; + + p->numa_faults_memory[i] += diff; + p->numa_faults_cpu[i] += f_diff; + faults += p->numa_faults_memory[i]; + p->total_numa_faults += diff; + if (p->numa_group) { + /* safe because we can only change our own group */ + p->numa_group->faults[i] += diff; + p->numa_group->faults_cpu[i] += f_diff; + p->numa_group->total_faults += diff; + group_faults += p->numa_group->faults[i]; + } + } - mutex_lock(&policy_mutex); + if (faults > max_faults) { + max_faults = faults; + max_nid = nid; + } - old_val = *data; + if (group_faults > max_group_faults) { + max_group_faults = group_faults; + max_group_nid = nid; + } + } - ret = proc_dointvec(table, write, buffer, lenp, ppos); - if (ret || !write || (write && (old_val == *data))) - goto done; + update_task_scan_period(p, fault_types[0], fault_types[1]); - if (invalid_value(data)) { - *data = old_val; - ret = -EINVAL; - goto done; + if (p->numa_group) { + update_numa_active_node_mask(p->numa_group); + spin_unlock_irq(group_lock); + max_nid = max_group_nid; } - reset_all_window_stats(0, 0); + if (max_faults) { + /* Set the new preferred node */ + if (max_nid != p->numa_preferred_nid) + sched_setnuma(p, max_nid); -done: - mutex_unlock(&policy_mutex); + if (task_node(p) != p->numa_preferred_nid) + numa_migrate_preferred(p); + } +} - return ret; +static inline int get_numa_group(struct numa_group *grp) +{ + return atomic_inc_not_zero(&grp->refcount); } -/* - * Convert percentage value into absolute form. This will avoid div() operation - * in fast path, to convert task load in percentage scale. - */ -int sched_hmp_proc_update_handler(struct ctl_table *table, int write, - void __user *buffer, size_t *lenp, - loff_t *ppos) +static inline void put_numa_group(struct numa_group *grp) { - int ret; - unsigned int old_val; - unsigned int *data = (unsigned int *)table->data; - int update_min_nice = 0; + if (atomic_dec_and_test(&grp->refcount)) + kfree_rcu(grp, rcu); +} - mutex_lock(&policy_mutex); +static void task_numa_group(struct task_struct *p, int cpupid, int flags, + int *priv) +{ + struct numa_group *grp, *my_grp; + struct task_struct *tsk; + bool join = false; + int cpu = cpupid_to_cpu(cpupid); + int i; - old_val = *data; + if (unlikely(!p->numa_group)) { + unsigned int size = sizeof(struct numa_group) + + 4*nr_node_ids*sizeof(unsigned long); - ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); + if (!grp) + return; - if (ret || !write || !sched_enable_hmp) - goto done; + atomic_set(&grp->refcount, 1); + spin_lock_init(&grp->lock); + INIT_LIST_HEAD(&grp->task_list); + grp->gid = p->pid; + /* Second half of the array tracks nids where faults happen */ + grp->faults_cpu = grp->faults + NR_NUMA_HINT_FAULT_TYPES * + nr_node_ids; - if (write && (old_val == *data)) - goto done; + node_set(task_node(current), grp->active_nodes); - if (data == &sysctl_sched_min_runtime) { - sched_min_runtime = ((u64) sysctl_sched_min_runtime) * 1000; - goto done; - } + for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) + grp->faults[i] = p->numa_faults_memory[i]; - if (data == (unsigned int *)&sysctl_sched_upmigrate_min_nice) { - if ((*(int *)data) < -20 || (*(int *)data) > 19) { - *data = old_val; - ret = -EINVAL; - goto done; - } - update_min_nice = 1; - } else { - /* all tunables other than min_nice are in percentage */ - if (sysctl_sched_downmigrate_pct > - sysctl_sched_upmigrate_pct || *data > 100) { - *data = old_val; - ret = -EINVAL; - goto done; - } + grp->total_faults = p->total_numa_faults; + + list_add(&p->numa_entry, &grp->task_list); + grp->nr_tasks++; + rcu_assign_pointer(p->numa_group, grp); } + rcu_read_lock(); + tsk = ACCESS_ONCE(cpu_rq(cpu)->curr); + + if (!cpupid_match_pid(tsk, cpupid)) + goto no_join; + + grp = rcu_dereference(tsk->numa_group); + if (!grp) + goto no_join; + + my_grp = p->numa_group; + if (grp == my_grp) + goto no_join; + /* - * Big/Small task tunable change will need to re-classify tasks on - * runqueue as big and small and set their counters appropriately. - * sysctl interface affects secondary variables (*_pct), which is then - * "atomically" carried over to the primary variables. Atomic change - * includes taking runqueue lock of all online cpus and re-initiatizing - * their big/small counter values based on changed criteria. + * Only join the other group if its bigger; if we're the bigger group, + * the other task will join us. */ - if ((data == &sysctl_sched_upmigrate_pct || - data == &sysctl_sched_small_task_pct || update_min_nice)) { - get_online_cpus(); - pre_big_small_task_count_change(cpu_online_mask); - } + if (my_grp->nr_tasks > grp->nr_tasks) + goto no_join; - set_hmp_defaults(); + /* + * Tie-break on the grp address. + */ + if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) + goto no_join; - if ((data == &sysctl_sched_upmigrate_pct || - data == &sysctl_sched_small_task_pct || update_min_nice)) { - post_big_small_task_count_change(cpu_online_mask); - put_online_cpus(); - } + /* Always join threads in the same process. */ + if (tsk->mm == current->mm) + join = true; -done: - mutex_unlock(&policy_mutex); - return ret; -} + /* Simple filter to avoid false positives due to PID collisions */ + if (flags & TNF_SHARED) + join = true; -/* - * Reset balance_interval at all sched_domain levels of given cpu, so that it - * honors kick. - */ -static inline void reset_balance_interval(int cpu) -{ - struct sched_domain *sd; + /* Update priv based on whether false sharing was detected */ + *priv = !join; - if (cpu >= nr_cpu_ids) - return; + if (join && !get_numa_group(grp)) + goto no_join; - rcu_read_lock(); - for_each_domain(cpu, sd) - sd->balance_interval = 0; rcu_read_unlock(); -} -static inline int find_new_hmp_ilb(int call_cpu, int type) -{ - int i; - int best_cpu = nr_cpu_ids; - struct sched_domain *sd; - int min_cost = INT_MAX, cost; - struct rq *src_rq = cpu_rq(call_cpu); - struct rq *dst_rq; + if (!join) + return; - rcu_read_lock(); + BUG_ON(irqs_disabled()); + double_lock_irq(&my_grp->lock, &grp->lock); - /* Pick an idle cpu "closest" to call_cpu */ - for_each_domain(call_cpu, sd) { - for_each_cpu(i, sched_domain_span(sd)) { - dst_rq = cpu_rq(i); - if (!idle_cpu(i) || (type == NOHZ_KICK_RESTRICT - && dst_rq->capacity > src_rq->capacity)) - continue; + for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { + my_grp->faults[i] -= p->numa_faults_memory[i]; + grp->faults[i] += p->numa_faults_memory[i]; + } + my_grp->total_faults -= p->total_numa_faults; + grp->total_faults += p->total_numa_faults; - cost = power_cost_at_freq(i, min_max_freq); - if (cost < min_cost) { - best_cpu = i; - min_cost = cost; - } - } + list_move(&p->numa_entry, &grp->task_list); + my_grp->nr_tasks--; + grp->nr_tasks++; - if (best_cpu < nr_cpu_ids) - break; - } + spin_unlock(&my_grp->lock); + spin_unlock_irq(&grp->lock); + rcu_assign_pointer(p->numa_group, grp); + + put_numa_group(my_grp); + return; + +no_join: rcu_read_unlock(); + return; +} + +void task_numa_free(struct task_struct *p) +{ + struct numa_group *grp = p->numa_group; + void *numa_faults = p->numa_faults_memory; + unsigned long flags; + int i; + + if (grp) { + spin_lock_irqsave(&grp->lock, flags); + for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) + grp->faults[i] -= p->numa_faults_memory[i]; + grp->total_faults -= p->total_numa_faults; - reset_balance_interval(best_cpu); + list_del(&p->numa_entry); + grp->nr_tasks--; + spin_unlock_irqrestore(&grp->lock, flags); + RCU_INIT_POINTER(p->numa_group, NULL); + put_numa_group(grp); + } - return best_cpu; + p->numa_faults_memory = NULL; + p->numa_faults_buffer_memory = NULL; + p->numa_faults_cpu= NULL; + p->numa_faults_buffer_cpu = NULL; + kfree(numa_faults); } /* - * For the current task's CPU, we don't check whether there are - * multiple tasks. Just see if running the task on another CPU is - * lower power than running only this task on the current CPU. This is - * not the most accurate model, but we should be load balanced most of - * the time anyway. */ -static int lower_power_cpu_available(struct task_struct *p, int cpu) + * Got a PROT_NONE fault for a page on @node. + */ +void task_numa_fault(int mem_node, int pages, int migrated) { - int i; - int lowest_power_cpu = task_cpu(p); - int lowest_power = power_cost(scale_load_to_cpu(task_load(p), - lowest_power_cpu), lowest_power_cpu); - struct cpumask search_cpus; - struct rq *rq = cpu_rq(cpu); + struct task_struct *p = current; + + if (!sched_feat_numa(NUMA)) + return; + + /* FIXME: Allocate task-specific structure for placement policy here */ /* - * This function should be called only when task 'p' fits in the current - * CPU which can be ensured by task_will_fit() prior to this. + * If pages are properly placed (did not migrate) then scan slower. + * This is reset periodically in case of phase changes */ - cpumask_and(&search_cpus, tsk_cpus_allowed(p), cpu_online_mask); - cpumask_and(&search_cpus, &search_cpus, &rq->freq_domain_cpumask); - cpumask_clear_cpu(lowest_power_cpu, &search_cpus); - - /* Is a lower-powered idle CPU available which will fit this task? */ - for_each_cpu(i, &search_cpus) { - if (idle_cpu(i)) { - int cost = - power_cost(scale_load_to_cpu(task_load(p), i), i); - if (cost < lowest_power) { - lowest_power_cpu = i; - lowest_power = cost; - } - } - } + if (!migrated) + p->numa_scan_period = min(sysctl_numa_balancing_scan_period_max, + p->numa_scan_period + jiffies_to_msecs(10)); - return (lowest_power_cpu != task_cpu(p)); + task_numa_placement(p); } -static inline int is_task_migration_throttled(struct task_struct *p); +static void reset_ptenuma_scan(struct task_struct *p) +{ + ACCESS_ONCE(p->mm->numa_scan_seq)++; + p->mm->numa_scan_offset = 0; +} /* - * Check if a task is on the "wrong" cpu (i.e its current cpu is not the ideal - * cpu as per its demand or priority) - * - * Returns reason why task needs to be migrated + * The expensive part of numa migration is done from task_work context. + * Triggered from task_tick_numa(). */ -static inline int migration_needed(struct rq *rq, struct task_struct *p) +void task_numa_work(struct callback_head *work) { - int nice = TASK_NICE(p); + unsigned long migrate, next_scan, now = jiffies; + struct task_struct *p = current; + struct mm_struct *mm = p->mm; + struct vm_area_struct *vma; + unsigned long start, end; + unsigned long nr_pte_updates = 0; + long pages; - if (!sched_enable_hmp || p->state != TASK_RUNNING) - return 0; + WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work)); - /* No need to migrate task that is about to be throttled */ - if (task_will_be_throttled(p)) - return 0; + work->next = work; /* protect against double add */ + /* + * Who cares about NUMA placement when they're dying. + * + * NOTE: make sure not to dereference p->mm before this check, + * exit_task_work() happens _after_ exit_mm() so we could be called + * without p->mm even though we still had it when we enqueued this + * work. + */ + if (p->flags & PF_EXITING) + return; - if (sched_boost()) { - if (rq->capacity != max_capacity) - return UP_MIGRATION; + if (!mm->numa_next_scan) { + mm->numa_next_scan = now + + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); + } - return 0; + /* + * Enforce maximal scan/migration frequency.. + */ + migrate = mm->numa_next_scan; + if (time_before(now, migrate)) + return; + + if (p->numa_scan_period == 0) { + p->numa_scan_period_max = task_scan_max(p); + p->numa_scan_period = task_scan_min(p); } - if (is_small_task(p)) - return 0; + next_scan = now + msecs_to_jiffies(p->numa_scan_period); + if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate) + return; - if (sched_cpu_high_irqload(cpu_of(rq))) - return IRQLOAD_MIGRATION; + /* + * Delay this task enough that another task of this mm will likely win + * the next time around. + */ + p->node_stamp += 2 * TICK_NSEC; - if ((nice > sched_upmigrate_min_nice || upmigrate_discouraged(p)) && - rq->capacity > min_capacity) - return DOWN_MIGRATION; + start = mm->numa_scan_offset; + pages = sysctl_numa_balancing_scan_size; + pages <<= 20 - PAGE_SHIFT; /* MB in pages */ + if (!pages) + return; - if (!task_will_fit(p, cpu_of(rq))) - return UP_MIGRATION; + down_read(&mm->mmap_sem); + vma = find_vma(mm, start); + if (!vma) { + reset_ptenuma_scan(p); + start = 0; + vma = mm->mmap; + } + for (; vma; vma = vma->vm_next) { + if (!vma_migratable(vma) || !vma_policy_mof(vma) || + is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { + continue; + } - if (sysctl_sched_enable_power_aware && - !is_task_migration_throttled(p) && - is_cpu_throttling_imminent(cpu_of(rq)) && - lower_power_cpu_available(p, cpu_of(rq))) - return EA_MIGRATION; + /* + * Shared library pages mapped by multiple processes are not + * migrated as it is expected they are cache replicated. Avoid + * hinting faults in read-only file-backed mappings or the vdso + * as migrating the pages will be of marginal benefit. + */ + if (!vma->vm_mm || + (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) + continue; - return 0; -} + /* + * Skip inaccessible VMAs to avoid any confusion between + * PROT_NONE and NUMA hinting ptes + */ + if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) + continue; -static DEFINE_RAW_SPINLOCK(migration_lock); + do { + start = max(start, vma->vm_start); + end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); + end = min(end, vma->vm_end); + nr_pte_updates += change_prot_numa(vma, start, end); -static inline int -kick_active_balance(struct rq *rq, struct task_struct *p, int new_cpu) -{ - unsigned long flags; - int rc = 0; + /* + * Scan sysctl_numa_balancing_scan_size but ensure that + * at least one PTE is updated so that unused virtual + * address space is quickly skipped. + */ + if (nr_pte_updates) + pages -= (end - start) >> PAGE_SHIFT; - /* Invoke active balance to force migrate currently running task */ - raw_spin_lock_irqsave(&rq->lock, flags); - if (!rq->active_balance) { - rq->active_balance = 1; - rq->push_cpu = new_cpu; - get_task_struct(p); - rq->push_task = p; - rc = 1; + start = end; + if (pages <= 0) + goto out; + + cond_resched(); + } while (end != vma->vm_end); } - raw_spin_unlock_irqrestore(&rq->lock, flags); - return rc; +out: + /* + * It is possible to reach the end of the VMA list but the last few + * VMAs are not guaranteed to the vma_migratable. If they are not, we + * would find the !migratable VMA on the next scan but not reset the + * scanner to the start so check it now. + */ + if (vma) + mm->numa_scan_offset = start; + else + reset_ptenuma_scan(p); + up_read(&mm->mmap_sem); } /* - * Check if currently running task should be migrated to a better cpu. - * - * Todo: Effect this via changes to nohz_balancer_kick() and load balance? + * Drive the periodic memory faults.. */ -void check_for_migration(struct rq *rq, struct task_struct *p) +void task_tick_numa(struct rq *rq, struct task_struct *curr) { - int cpu = cpu_of(rq), new_cpu; - int active_balance = 0, reason; + struct callback_head *work = &curr->numa_work; + u64 period, now; - reason = migration_needed(rq, p); - if (!reason) + /* + * We don't care about NUMA placement if we don't have memory. + */ + if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work) return; - raw_spin_lock(&migration_lock); - new_cpu = select_best_cpu(p, cpu, reason, 0); - - if (new_cpu != cpu) { - active_balance = kick_active_balance(rq, p, new_cpu); - if (active_balance) - mark_reserved(new_cpu); - } + /* + * Using runtime rather than walltime has the dual advantage that + * we (mostly) drive the selection from busy threads and that the + * task needs to have done some actual work before we bother with + * NUMA placement. + */ + now = curr->se.sum_exec_runtime; + period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; - raw_spin_unlock(&migration_lock); + if (now - curr->node_stamp > period) { + if (!curr->node_stamp) + curr->numa_scan_period = task_scan_min(curr); + curr->node_stamp += period; - if (active_balance) - stop_one_cpu_nowait(cpu, active_load_balance_cpu_stop, rq, - &rq->active_balance_work); + if (!time_before(jiffies, curr->mm->numa_next_scan)) { + init_task_work(work, task_numa_work); /* TODO: move this into sched_fork() */ + task_work_add(curr, work, true); + } + } } - -static inline int nr_big_tasks(struct rq *rq) +#else +static void task_tick_numa(struct rq *rq, struct task_struct *curr) { - return rq->hmp_stats.nr_big_tasks; } -static inline int is_cpu_throttling_imminent(int cpu) +static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p) { - int throttling = 0; - struct cpu_pwr_stats *per_cpu_info; - - if (sched_feat(FORCE_CPU_THROTTLING_IMMINENT)) - return 1; +} - per_cpu_info = get_cpu_pwr_stats(); - if (per_cpu_info) - throttling = per_cpu_info[cpu].throttling; - return throttling; +static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p) +{ } +#endif /* CONFIG_NUMA_BALANCING */ -static inline int is_task_migration_throttled(struct task_struct *p) +static void +account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) { - u64 delta = sched_clock() - p->run_start; + update_load_add(&cfs_rq->load, se->load.weight); + if (!parent_entity(se)) + update_load_add(&rq_of(cfs_rq)->load, se->load.weight); +#ifdef CONFIG_SMP + if (entity_is_task(se)) { + struct rq *rq = rq_of(cfs_rq); - return delta < sched_min_runtime; + account_numa_enqueue(rq, task_of(se)); + list_add(&se->group_node, &rq->cfs_tasks); + } +#endif + cfs_rq->nr_running++; } -unsigned int cpu_temp(int cpu) +static void +account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) { - struct cpu_pwr_stats *per_cpu_info = get_cpu_pwr_stats(); - if (per_cpu_info) - return per_cpu_info[cpu].temp; - else - return 0; + update_load_sub(&cfs_rq->load, se->load.weight); + if (!parent_entity(se)) + update_load_sub(&rq_of(cfs_rq)->load, se->load.weight); + if (entity_is_task(se)) { + account_numa_dequeue(rq_of(cfs_rq), task_of(se)); + list_del_init(&se->group_node); + } + cfs_rq->nr_running--; } -#else /* CONFIG_SCHED_HMP */ +#ifdef CONFIG_FAIR_GROUP_SCHED +# ifdef CONFIG_SMP +static inline long calc_tg_weight(struct task_group *tg, struct cfs_rq *cfs_rq) +{ + long tg_weight; -#define sysctl_sched_enable_power_aware 0 + /* + * Use this CPU's actual weight instead of the last load_contribution + * to gain a more accurate current total weight. See + * update_cfs_rq_load_contribution(). + */ + tg_weight = atomic_long_read(&tg->load_avg); + tg_weight -= cfs_rq->tg_load_contrib; + tg_weight += cfs_rq->load.weight; -static inline int task_will_fit(struct task_struct *p, int cpu) -{ - return 1; + return tg_weight; } -static inline int select_best_cpu(struct task_struct *p, int target, - int reason, int sync) +static long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg) { - return 0; -} + long tg_weight, load, shares; -static inline int find_new_hmp_ilb(int call_cpu, int type) -{ - return 0; -} + tg_weight = calc_tg_weight(tg, cfs_rq); + load = cfs_rq->load.weight; -static inline int power_cost(u64 task_load, int cpu) -{ - return SCHED_POWER_SCALE; -} + shares = (tg->shares * load); + if (tg_weight) + shares /= tg_weight; -static inline int -spill_threshold_crossed(u64 task_load, u64 cpu_load, struct rq *rq) -{ - return 0; -} + if (shares < MIN_SHARES) + shares = MIN_SHARES; + if (shares > tg->shares) + shares = tg->shares; -static inline int mostly_idle_cpu(int cpu) -{ - return 0; + return shares; } - -static inline int sched_boost(void) +# else /* CONFIG_SMP */ +static inline long calc_cfs_shares(struct cfs_rq *cfs_rq, struct task_group *tg) { - return 0; + return tg->shares; } - -static inline int is_small_task(struct task_struct *p) +# endif /* CONFIG_SMP */ +static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight) { - return 0; -} + if (se->on_rq) { + /* commit outstanding execution time */ + if (cfs_rq->curr == se) + update_curr(cfs_rq); + account_entity_dequeue(cfs_rq, se); + } -static inline int is_big_task(struct task_struct *p) -{ - return 0; -} + update_load_set(&se->load, weight); -static inline int nr_big_tasks(struct rq *rq) -{ - return 0; + if (se->on_rq) + account_entity_enqueue(cfs_rq, se); } -static inline int is_cpu_throttling_imminent(int cpu) -{ - return 0; -} +static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); -static inline int is_task_migration_throttled(struct task_struct *p) +static void update_cfs_shares(struct cfs_rq *cfs_rq) { - return 0; -} + struct task_group *tg; + struct sched_entity *se; + long shares; + + tg = cfs_rq->tg; + se = tg->se[cpu_of(rq_of(cfs_rq))]; + if (!se || throttled_hierarchy(cfs_rq)) + return; +#ifndef CONFIG_SMP + if (likely(se->load.weight == tg->shares)) + return; +#endif + shares = calc_cfs_shares(cfs_rq, tg); -unsigned int cpu_temp(int cpu) + reweight_entity(cfs_rq_of(se), se, shares); +} +#else /* CONFIG_FAIR_GROUP_SCHED */ +static inline void update_cfs_shares(struct cfs_rq *cfs_rq) { - return 0; } +#endif /* CONFIG_FAIR_GROUP_SCHED */ -static inline void -inc_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) { } -static inline void -dec_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) { } - -static inline void -inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p) { } - -static inline void -dec_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p) { } +#ifdef CONFIG_SMP +/* + * We choose a half-life close to 1 scheduling period. + * Note: The tables below are dependent on this value. + */ +#define LOAD_AVG_PERIOD 32 +#define LOAD_AVG_MAX 47742 /* maximum possible load avg */ +#define LOAD_AVG_MAX_N 345 /* number of full periods to produce LOAD_MAX_AVG */ -#endif /* CONFIG_SCHED_HMP */ +/* Precomputed fixed inverse multiplies for multiplication by y^n */ +static const u32 runnable_avg_yN_inv[] = { + 0xffffffff, 0xfa83b2da, 0xf5257d14, 0xefe4b99a, 0xeac0c6e6, 0xe5b906e6, + 0xe0ccdeeb, 0xdbfbb796, 0xd744fcc9, 0xd2a81d91, 0xce248c14, 0xc9b9bd85, + 0xc5672a10, 0xc12c4cc9, 0xbd08a39e, 0xb8fbaf46, 0xb504f333, 0xb123f581, + 0xad583ee9, 0xa9a15ab4, 0xa5fed6a9, 0xa2704302, 0x9ef5325f, 0x9b8d39b9, + 0x9837f050, 0x94f4efa8, 0x91c3d373, 0x8ea4398a, 0x8b95c1e3, 0x88980e80, + 0x85aac367, 0x82cd8698, +}; -#ifdef CONFIG_SCHED_HMP +/* + * Precomputed \Sum y^k { 1<=k<=n }. These are floor(true_value) to prevent + * over-estimates when re-combining. + */ +static const u32 runnable_avg_yN_sum[] = { + 0, 1002, 1982, 2941, 3880, 4798, 5697, 6576, 7437, 8279, 9103, + 9909,10698,11470,12226,12966,13690,14398,15091,15769,16433,17082, + 17718,18340,18949,19545,20128,20698,21256,21802,22336,22859,23371, +}; -void init_new_task_load(struct task_struct *p) +/* + * Approximate: + * val * y^n, where y^32 ~= 0.5 (~1 scheduling period) + */ +static __always_inline u64 decay_load(u64 val, u64 n) { - int i; - u32 init_load_windows = sched_init_task_load_windows; - u32 init_load_pelt = sched_init_task_load_pelt; - u32 init_load_pct = current->init_load_pct; + unsigned int local_n; - /* Note: child's init_load_pct itself would be 0 */ - memset(&p->ravg, 0, sizeof(struct ravg)); - p->se.avg.decay_count = 0; + if (!n) + return val; + else if (unlikely(n > LOAD_AVG_PERIOD * 63)) + return 0; - if (init_load_pct) { - init_load_pelt = div64_u64((u64)init_load_pct * - (u64)LOAD_AVG_MAX, 100); - init_load_windows = div64_u64((u64)init_load_pct * - (u64)sched_ravg_window, 100); - } + /* after bounds checking we can collapse to 32-bit */ + local_n = n; - p->ravg.demand = init_load_windows; - for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i) - p->ravg.sum_history[i] = init_load_windows; + /* + * As y^PERIOD = 1/2, we can combine + * y^n = 1/2^(n/PERIOD) * y^(n%PERIOD) + * With a look-up table which covers y^n (n= LOAD_AVG_PERIOD)) { + val >>= local_n / LOAD_AVG_PERIOD; + local_n %= LOAD_AVG_PERIOD; + } - p->se.avg.runnable_avg_period = - init_load_pelt ? LOAD_AVG_MAX : 0; - p->se.avg.runnable_avg_sum = init_load_pelt; - p->se.avg.runnable_avg_sum_scaled = init_load_pelt; + val *= runnable_avg_yN_inv[local_n]; + /* We don't use SRR here since we always want to round down. */ + return val >> 32; } -#else /* CONFIG_SCHED_HMP */ - -#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED) - -void init_new_task_load(struct task_struct *p) +/* + * For updates fully spanning n periods, the contribution to runnable + * average will be: \Sum 1024*y^n + * + * We can compute this reasonably efficiently by combining: + * y^PERIOD = 1/2 with precomputed \Sum 1024*y^n {for n se.avg.decay_count = 0; - p->se.avg.runnable_avg_period = 0; - p->se.avg.runnable_avg_sum = 0; -} + u32 contrib = 0; -#else /* CONFIG_SMP && CONFIG_FAIR_GROUP_SCHED */ + if (likely(n <= LOAD_AVG_PERIOD)) + return runnable_avg_yN_sum[n]; + else if (unlikely(n >= LOAD_AVG_MAX_N)) + return LOAD_AVG_MAX; -void init_new_task_load(struct task_struct *p) -{ -} + /* Compute \Sum k^n combining precomputed values for k^i, \Sum k^j */ + do { + contrib /= 2; /* y^LOAD_AVG_PERIOD = 1/2 */ + contrib += runnable_avg_yN_sum[LOAD_AVG_PERIOD]; -#endif /* CONFIG_SMP && CONFIG_FAIR_GROUP_SCHED */ + n -= LOAD_AVG_PERIOD; + } while (n > LOAD_AVG_PERIOD); -#endif /* CONFIG_SCHED_HMP */ + contrib = decay_load(contrib, n); + return contrib + runnable_avg_yN_sum[n]; +} /* * We can represent the historical contribution to runnable average as the @@ -3106,13 +2279,21 @@ void init_new_task_load(struct task_struct *p) * load_avg = u_0` + y*(u_0 + u_1*y + u_2*y^2 + ... ) * = u_0 + u_1*y + u_2*y^2 + ... [re-labeling u_i --> u_{i+1}] */ -static __always_inline int __update_entity_runnable_avg(int cpu, u64 now, +static __always_inline int __update_entity_runnable_avg(u64 now, int cpu, struct sched_avg *sa, - int runnable) + int runnable, + int running) { - u64 delta, periods; - u32 runnable_contrib; - int delta_w, decayed = 0; + u64 delta, scaled_delta, periods; + u32 runnable_contrib, scaled_runnable_contrib; + int delta_w, scaled_delta_w, decayed = 0; + unsigned long scale_freq = arch_scale_freq_capacity(NULL, cpu); + unsigned long scale_cpu = arch_scale_cpu_capacity(NULL, cpu); + +#define TK_PERIOD_SHIFT 10 +#define TK_PERIOD (1<last_runnable_update; /* @@ -3128,14 +2309,14 @@ static __always_inline int __update_entity_runnable_avg(int cpu, u64 now, * Use 1024ns as the unit of measurement since it's a reasonable * approximation of 1us and fast to compute. */ - delta >>= 10; + delta >>= TK_PERIOD_SHIFT; if (!delta) return 0; sa->last_runnable_update = now; /* delta_w is the amount already accumulated against our next period */ - delta_w = sa->runnable_avg_period % 1024; - if (delta + delta_w >= 1024) { + delta_w = sa->avg_period % TK_PERIOD; + if (delta + delta_w >= TK_PERIOD) { /* period roll-over */ decayed = 1; @@ -3144,40 +2325,60 @@ static __always_inline int __update_entity_runnable_avg(int cpu, u64 now, * out how much from delta we need to complete the current * period and accrue it. */ - delta_w = 1024 - delta_w; - if (runnable) { - sa->runnable_avg_sum += delta_w; - add_to_scaled_stat(cpu, sa, delta_w); - } - sa->runnable_avg_period += delta_w; + delta_w = TK_PERIOD - delta_w; + scaled_delta_w = (delta_w * scale_freq) >> SCHED_CAPACITY_SHIFT; + + if (runnable) + sa->runnable_avg_sum += scaled_delta_w; + + scaled_delta_w *= scale_cpu; + scaled_delta_w >>= SCHED_CAPACITY_SHIFT; + + if (running) + sa->running_avg_sum += scaled_delta_w; + sa->avg_period += delta_w; delta -= delta_w; /* Figure out how many additional periods this update spans */ - periods = delta / 1024; - delta %= 1024; + periods = delta / TK_PERIOD; + delta %= TK_PERIOD; sa->runnable_avg_sum = decay_load(sa->runnable_avg_sum, periods + 1); - decay_scaled_stat(sa, periods + 1); - sa->runnable_avg_period = decay_load(sa->runnable_avg_period, + sa->running_avg_sum = decay_load(sa->running_avg_sum, + periods + 1); + sa->avg_period = decay_load(sa->avg_period, periods + 1); /* Efficiently calculate \sum (1..n_period) 1024*y^i */ runnable_contrib = __compute_runnable_contrib(periods); - if (runnable) { - sa->runnable_avg_sum += runnable_contrib; - add_to_scaled_stat(cpu, sa, runnable_contrib); - } - sa->runnable_avg_period += runnable_contrib; + scaled_runnable_contrib = (runnable_contrib * scale_freq) + >> SCHED_CAPACITY_SHIFT; + + if (runnable) + sa->runnable_avg_sum += scaled_runnable_contrib; + + scaled_runnable_contrib *= scale_cpu; + scaled_runnable_contrib >>= SCHED_CAPACITY_SHIFT; + + if (running) + sa->running_avg_sum += scaled_runnable_contrib; + sa->avg_period += runnable_contrib; } /* Remainder of delta accrued against u_0` */ - if (runnable) { - sa->runnable_avg_sum += delta; - add_to_scaled_stat(cpu, sa, delta); - } - sa->runnable_avg_period += delta; + scaled_delta = (delta * scale_freq) >> SCHED_CAPACITY_SHIFT; + + if (runnable) + sa->runnable_avg_sum += scaled_delta; + + scaled_delta *= scale_cpu; + scaled_delta >>= SCHED_CAPACITY_SHIFT; + + if (running) + sa->running_avg_sum += scaled_delta; + sa->avg_period += delta; return decayed; } @@ -3189,13 +2390,13 @@ static inline u64 __synchronize_entity_decay(struct sched_entity *se) u64 decays = atomic64_read(&cfs_rq->decay_counter); decays -= se->avg.decay_count; - if (!decays) { - se->avg.decay_count = 0; + se->avg.decay_count = 0; + if (!decays) return 0; - } se->avg.load_avg_contrib = decay_load(se->avg.load_avg_contrib, decays); - se->avg.decay_count = 0; + se->avg.utilization_avg_contrib = + decay_load(se->avg.utilization_avg_contrib, decays); return decays; } @@ -3205,13 +2406,16 @@ static inline void __update_cfs_rq_tg_load_contrib(struct cfs_rq *cfs_rq, int force_update) { struct task_group *tg = cfs_rq->tg; - s64 tg_contrib; + long tg_contrib; tg_contrib = cfs_rq->runnable_load_avg + cfs_rq->blocked_load_avg; tg_contrib -= cfs_rq->tg_load_contrib; - if (force_update || abs64(tg_contrib) > cfs_rq->tg_load_contrib / 8) { - atomic64_add(tg_contrib, &tg->load_avg); + if (!tg_contrib) + return; + + if (force_update || abs(tg_contrib) > cfs_rq->tg_load_contrib / 8) { + atomic_long_add(tg_contrib, &tg->load_avg); cfs_rq->tg_load_contrib += tg_contrib; } } @@ -3227,8 +2431,8 @@ static inline void __update_tg_runnable_avg(struct sched_avg *sa, long contrib; /* The fraction of a cpu used by this cfs_rq */ - contrib = div_u64(sa->runnable_avg_sum << NICE_0_SHIFT, - sa->runnable_avg_period + 1); + contrib = div_u64((u64)sa->runnable_avg_sum << NICE_0_SHIFT, + sa->avg_period + 1); contrib -= cfs_rq->tg_runnable_contrib; if (abs(contrib) > cfs_rq->tg_runnable_contrib / 64) { @@ -3246,8 +2450,8 @@ static inline void __update_group_entity_contrib(struct sched_entity *se) u64 contrib; contrib = cfs_rq->tg_load_contrib * tg->shares; - se->avg.load_avg_contrib = div64_u64(contrib, - atomic64_read(&tg->load_avg) + 1); + se->avg.load_avg_contrib = div_u64(contrib, + atomic_long_read(&tg->load_avg) + 1); /* * For group entities we need to compute a correction term in the case @@ -3278,13 +2482,21 @@ static inline void __update_group_entity_contrib(struct sched_entity *se) se->avg.load_avg_contrib >>= NICE_0_SHIFT; } } -#else + +static inline void update_rq_runnable_avg(struct rq *rq, int runnable) +{ + __update_entity_runnable_avg(rq_clock_task(rq), cpu_of(rq), &rq->avg, + runnable, runnable); + __update_tg_runnable_avg(&rq->avg, &rq->cfs); +} +#else /* CONFIG_FAIR_GROUP_SCHED */ static inline void __update_cfs_rq_tg_load_contrib(struct cfs_rq *cfs_rq, int force_update) {} static inline void __update_tg_runnable_avg(struct sched_avg *sa, struct cfs_rq *cfs_rq) {} static inline void __update_group_entity_contrib(struct sched_entity *se) {} -#endif +static inline void update_rq_runnable_avg(struct rq *rq, int runnable) {} +#endif /* CONFIG_FAIR_GROUP_SCHED */ static inline void __update_task_entity_contrib(struct sched_entity *se) { @@ -3292,7 +2504,7 @@ static inline void __update_task_entity_contrib(struct sched_entity *se) /* avoid overflowing a 32-bit type w/ SCHED_LOAD_SCALE */ contrib = se->avg.runnable_avg_sum * scale_load_down(se->load.weight); - contrib /= (se->avg.runnable_avg_period + 1); + contrib /= (se->avg.avg_period + 1); se->avg.load_avg_contrib = scale_load(contrib); } @@ -3311,6 +2523,31 @@ static long __update_entity_load_avg_contrib(struct sched_entity *se) return se->avg.load_avg_contrib - old_contrib; } + +static inline void __update_task_entity_utilization(struct sched_entity *se) +{ + u32 contrib; + + /* avoid overflowing a 32-bit type w/ SCHED_LOAD_SCALE */ + contrib = se->avg.running_avg_sum * scale_load_down(SCHED_LOAD_SCALE); + contrib /= (se->avg.avg_period + 1); + se->avg.utilization_avg_contrib = scale_load(contrib); +} + +static long __update_entity_utilization_avg_contrib(struct sched_entity *se) +{ + long old_contrib = se->avg.utilization_avg_contrib; + + if (entity_is_task(se)) + __update_task_entity_utilization(se); + else + se->avg.utilization_avg_contrib = + group_cfs_rq(se)->utilization_load_avg + + group_cfs_rq(se)->utilization_blocked_avg; + + return se->avg.utilization_avg_contrib - old_contrib; +} + static inline void subtract_blocked_load_contrib(struct cfs_rq *cfs_rq, long load_contrib) { @@ -3320,6 +2557,15 @@ static inline void subtract_blocked_load_contrib(struct cfs_rq *cfs_rq, cfs_rq->blocked_load_avg = 0; } +static inline void subtract_utilization_blocked_contrib(struct cfs_rq *cfs_rq, + long utilization_contrib) +{ + if (likely(utilization_contrib < cfs_rq->utilization_blocked_avg)) + cfs_rq->utilization_blocked_avg -= utilization_contrib; + else + cfs_rq->utilization_blocked_avg = 0; +} + static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq); /* Update a sched_entity's runnable average */ @@ -3327,38 +2573,42 @@ static inline void update_entity_load_avg(struct sched_entity *se, int update_cfs_rq) { struct cfs_rq *cfs_rq = cfs_rq_of(se); - long contrib_delta; - u64 now; + long contrib_delta, utilization_delta; int cpu = cpu_of(rq_of(cfs_rq)); - int decayed; + u64 now; /* * For a group entity we need to use their owned cfs_rq_clock_task() in * case they are the parent of a throttled hierarchy. */ - if (entity_is_task(se)) { + if (entity_is_task(se)) now = cfs_rq_clock_task(cfs_rq); - if (se->on_rq) - dec_hmp_sched_stats_fair(rq_of(cfs_rq), task_of(se)); - } else + else now = cfs_rq_clock_task(group_cfs_rq(se)); - decayed = __update_entity_runnable_avg(cpu, now, &se->avg, se->on_rq); - if (entity_is_task(se) && se->on_rq) - inc_hmp_sched_stats_fair(rq_of(cfs_rq), task_of(se)); - - if (!decayed) + if (!__update_entity_runnable_avg(now, cpu, &se->avg, se->on_rq, + cfs_rq->curr == se)) return; contrib_delta = __update_entity_load_avg_contrib(se); + utilization_delta = __update_entity_utilization_avg_contrib(se); + + if (entity_is_task(se)) + trace_sched_load_avg_task(task_of(se), &se->avg); if (!update_cfs_rq) return; - if (se->on_rq) + if (se->on_rq) { cfs_rq->runnable_load_avg += contrib_delta; - else + cfs_rq->utilization_load_avg += utilization_delta; + } else { subtract_blocked_load_contrib(cfs_rq, -contrib_delta); + subtract_utilization_blocked_contrib(cfs_rq, + -utilization_delta); + } + + trace_sched_load_avg_cpu(cpu, cfs_rq); } /* @@ -3374,14 +2624,21 @@ static void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq, int force_update) if (!decays && !force_update) return; - if (atomic64_read(&cfs_rq->removed_load)) { - u64 removed_load = atomic64_xchg(&cfs_rq->removed_load, 0); + if (atomic_long_read(&cfs_rq->removed_load)) { + unsigned long removed_load, removed_utilization; + removed_load = atomic_long_xchg(&cfs_rq->removed_load, 0); + removed_utilization = + atomic_long_xchg(&cfs_rq->removed_utilization, 0); subtract_blocked_load_contrib(cfs_rq, removed_load); + subtract_utilization_blocked_contrib(cfs_rq, + removed_utilization); } if (decays) { cfs_rq->blocked_load_avg = decay_load(cfs_rq->blocked_load_avg, decays); + cfs_rq->utilization_blocked_avg = + decay_load(cfs_rq->utilization_blocked_avg, decays); atomic64_add(decays, &cfs_rq->decay_counter); cfs_rq->last_decay = now; } @@ -3389,13 +2646,6 @@ static void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq, int force_update) __update_cfs_rq_tg_load_contrib(cfs_rq, force_update); } -static inline void update_rq_runnable_avg(struct rq *rq, int runnable) -{ - __update_entity_runnable_avg(cpu_of(rq), rq->clock_task, - &rq->avg, runnable); - __update_tg_runnable_avg(&rq->avg, &rq->cfs); -} - /* Add the load generated by se into cfs_rq's child load-average */ static inline void enqueue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, @@ -3405,9 +2655,13 @@ static inline void enqueue_entity_load_avg(struct cfs_rq *cfs_rq, * We track migrations using entity decay_count <= 0, on a wake-up * migration we use a negative decay count to track the remote decays * accumulated while sleeping. + * + * Newly forked tasks are enqueued with se->avg.decay_count == 0, they + * are seen by enqueue_entity_load_avg() as a migration with an already + * constructed load_avg_contrib. */ if (unlikely(se->avg.decay_count <= 0)) { - se->avg.last_runnable_update = rq_of(cfs_rq)->clock_task; + se->avg.last_runnable_update = rq_clock_task(rq_of(cfs_rq)); if (se->avg.decay_count) { /* * In a wake-up migration we have to approximate the @@ -3431,10 +2685,13 @@ static inline void enqueue_entity_load_avg(struct cfs_rq *cfs_rq, /* migrated tasks did not contribute to our blocked load */ if (wakeup) { subtract_blocked_load_contrib(cfs_rq, se->avg.load_avg_contrib); + subtract_utilization_blocked_contrib(cfs_rq, + se->avg.utilization_avg_contrib); update_entity_load_avg(se, 0); } cfs_rq->runnable_load_avg += se->avg.load_avg_contrib; + cfs_rq->utilization_load_avg += se->avg.utilization_avg_contrib; /* we force update consideration on load-balancer moves */ update_cfs_rq_blocked_load(cfs_rq, !wakeup); } @@ -3453,8 +2710,11 @@ static inline void dequeue_entity_load_avg(struct cfs_rq *cfs_rq, update_cfs_rq_blocked_load(cfs_rq, !sleep); cfs_rq->runnable_load_avg -= se->avg.load_avg_contrib; + cfs_rq->utilization_load_avg -= se->avg.utilization_avg_contrib; if (sleep) { cfs_rq->blocked_load_avg += se->avg.load_avg_contrib; + cfs_rq->utilization_blocked_avg += + se->avg.utilization_avg_contrib; se->avg.decay_count = atomic64_read(&cfs_rq->decay_counter); } /* migrations, e.g. sleep=0 leave decay_count == 0 */ } @@ -3479,7 +2739,10 @@ void idle_exit_fair(struct rq *this_rq) update_rq_runnable_avg(this_rq, 0); } -#else +static int idle_balance(struct rq *this_rq); + +#else /* CONFIG_SMP */ + static inline void update_entity_load_avg(struct sched_entity *se, int update_cfs_rq) {} static inline void update_rq_runnable_avg(struct rq *rq, int runnable) {} @@ -3492,149 +2755,12 @@ static inline void dequeue_entity_load_avg(struct cfs_rq *cfs_rq, static inline void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq, int force_update) {} -static inline void -inc_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) { } -static inline void -dec_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) { } - -#endif - -#ifdef CONFIG_SCHED_HMP - -/* Return task demand in percentage scale */ -unsigned int pct_task_load(struct task_struct *p) -{ - unsigned int load; - - load = div64_u64((u64)task_load(p) * 100, (u64)max_task_load()); - - return load; -} - -/* - * Add scaled version of 'delta' to runnable_avg_sum_scaled - * 'delta' is scaled in reference to "best" cpu - */ -static inline void -add_to_scaled_stat(int cpu, struct sched_avg *sa, u64 delta) -{ - struct rq *rq = cpu_rq(cpu); - int cur_freq = rq->cur_freq, max_freq = rq->max_freq; - int cpu_max_possible_freq = rq->max_possible_freq; - u64 scaled_delta; - int sf; - - if (!sched_enable_hmp) - return; - - if (unlikely(cur_freq > max_possible_freq || - (cur_freq == max_freq && - max_freq < cpu_max_possible_freq))) - cur_freq = max_possible_freq; - - scaled_delta = div64_u64(delta * cur_freq, max_possible_freq); - sf = (rq->efficiency * 1024) / max_possible_efficiency; - scaled_delta *= sf; - scaled_delta >>= 10; - sa->runnable_avg_sum_scaled += scaled_delta; -} - -static inline void decay_scaled_stat(struct sched_avg *sa, u64 periods) -{ - if (!sched_enable_hmp) - return; - - sa->runnable_avg_sum_scaled = - decay_load(sa->runnable_avg_sum_scaled, - periods); -} - -#ifdef CONFIG_CFS_BANDWIDTH - -static void init_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq) -{ - cfs_rq->hmp_stats.nr_big_tasks = 0; - cfs_rq->hmp_stats.nr_small_tasks = 0; - cfs_rq->hmp_stats.cumulative_runnable_avg = 0; -} - -static void inc_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra) -{ - inc_nr_big_small_task(&cfs_rq->hmp_stats, p); - if (change_cra) - inc_cumulative_runnable_avg(&cfs_rq->hmp_stats, p); -} - -static void dec_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra) -{ - dec_nr_big_small_task(&cfs_rq->hmp_stats, p); - if (change_cra) - dec_cumulative_runnable_avg(&cfs_rq->hmp_stats, p); -} - -static void inc_throttled_cfs_rq_hmp_stats(struct hmp_sched_stats *stats, - struct cfs_rq *cfs_rq) -{ - stats->nr_big_tasks += cfs_rq->hmp_stats.nr_big_tasks; - stats->nr_small_tasks += cfs_rq->hmp_stats.nr_small_tasks; - stats->cumulative_runnable_avg += - cfs_rq->hmp_stats.cumulative_runnable_avg; -} - -static void dec_throttled_cfs_rq_hmp_stats(struct hmp_sched_stats *stats, - struct cfs_rq *cfs_rq) -{ - stats->nr_big_tasks -= cfs_rq->hmp_stats.nr_big_tasks; - stats->nr_small_tasks -= cfs_rq->hmp_stats.nr_small_tasks; - stats->cumulative_runnable_avg -= - cfs_rq->hmp_stats.cumulative_runnable_avg; - - BUG_ON(stats->nr_big_tasks < 0 || stats->nr_small_tasks < 0 || - (s64)stats->cumulative_runnable_avg < 0); -} - -#else /* CONFIG_CFS_BANDWIDTH */ - -static inline void inc_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra) { } - -static inline void dec_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra) { } - -#endif /* CONFIG_CFS_BANDWIDTH */ - -#else /* CONFIG_SCHED_HMP */ - -static inline void -add_to_scaled_stat(int cpu, struct sched_avg *sa, u64 delta) -{ -} - -static inline void decay_scaled_stat(struct sched_avg *sa, u64 periods) -{ -} - -static inline void init_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq) { } - -static inline void inc_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra) { } - -static inline void dec_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, - struct task_struct *p, int change_cra) { } - -static inline void inc_throttled_cfs_rq_hmp_stats(struct hmp_sched_stats *stats, - struct cfs_rq *cfs_rq) -{ -} - -static inline void dec_throttled_cfs_rq_hmp_stats(struct hmp_sched_stats *stats, - struct cfs_rq *cfs_rq) +static inline int idle_balance(struct rq *rq) { + return 0; } -#endif /* CONFIG_SCHED_HMP */ +#endif /* CONFIG_SMP */ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) { @@ -3645,7 +2771,7 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) tsk = task_of(se); if (se->statistics.sleep_start) { - u64 delta = rq_of(cfs_rq)->clock - se->statistics.sleep_start; + u64 delta = rq_clock(rq_of(cfs_rq)) - se->statistics.sleep_start; if ((s64)delta < 0) delta = 0; @@ -3662,7 +2788,7 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) } } if (se->statistics.block_start) { - u64 delta = rq_of(cfs_rq)->clock - se->statistics.block_start; + u64 delta = rq_clock(rq_of(cfs_rq)) - se->statistics.block_start; if ((s64)delta < 0) delta = 0; @@ -3751,7 +2877,7 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { /* * Update the normalized vruntime before updating min_vruntime - * through callig update_curr(). + * through calling update_curr(). */ if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_WAKING)) se->vruntime += cfs_rq->min_vruntime; @@ -3785,10 +2911,10 @@ static void __clear_buddies_last(struct sched_entity *se) { for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (cfs_rq->last == se) - cfs_rq->last = NULL; - else + if (cfs_rq->last != se) break; + + cfs_rq->last = NULL; } } @@ -3796,10 +2922,10 @@ static void __clear_buddies_next(struct sched_entity *se) { for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (cfs_rq->next == se) - cfs_rq->next = NULL; - else + if (cfs_rq->next != se) break; + + cfs_rq->next = NULL; } } @@ -3807,10 +2933,10 @@ static void __clear_buddies_skip(struct sched_entity *se) { for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (cfs_rq->skip == se) - cfs_rq->skip = NULL; - else + if (cfs_rq->skip != se) break; + + cfs_rq->skip = NULL; } } @@ -3835,6 +2961,8 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) * Update run-time statistics of the 'current'. */ update_curr(cfs_rq); + if (entity_is_task(se) && task_of(se)->state == TASK_DEAD) + flags &= !DEQUEUE_SLEEP; dequeue_entity_load_avg(cfs_rq, se, flags & DEQUEUE_SLEEP); update_stats_dequeue(cfs_rq, se); @@ -3844,9 +2972,9 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) struct task_struct *tsk = task_of(se); if (tsk->state & TASK_INTERRUPTIBLE) - se->statistics.sleep_start = rq_of(cfs_rq)->clock; + se->statistics.sleep_start = rq_clock(rq_of(cfs_rq)); if (tsk->state & TASK_UNINTERRUPTIBLE) - se->statistics.block_start = rq_of(cfs_rq)->clock; + se->statistics.block_start = rq_clock(rq_of(cfs_rq)); } #endif } @@ -3886,7 +3014,7 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) ideal_runtime = sched_slice(cfs_rq, curr); delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; if (delta_exec > ideal_runtime) { - resched_task(rq_of(cfs_rq)->curr); + resched_curr(rq_of(cfs_rq)); /* * The current task ran long enough, ensure it doesn't get * re-elected due to buddy favours. @@ -3910,7 +3038,7 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) return; if (delta > ideal_runtime) - resched_task(rq_of(cfs_rq)->curr); + resched_curr(rq_of(cfs_rq)); } static void @@ -3925,6 +3053,7 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) */ update_stats_wait_end(cfs_rq, se); __dequeue_entity(cfs_rq, se); + update_entity_load_avg(se, 1); } update_stats_curr_start(cfs_rq, se); @@ -3953,17 +3082,36 @@ wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se); * 3) pick the "last" process, for cache locality * 4) do not run the "skip" process, if something else is available */ -static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) +static struct sched_entity * +pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr) { - struct sched_entity *se = __pick_first_entity(cfs_rq); - struct sched_entity *left = se; + struct sched_entity *left = __pick_first_entity(cfs_rq); + struct sched_entity *se; + + /* + * If curr is set we have to see if its left of the leftmost entity + * still in the tree, provided there was anything in the tree at all. + */ + if (!left || (curr && entity_before(curr, left))) + left = curr; + + se = left; /* ideally we run the leftmost entity */ /* * Avoid running the skip buddy, if running something else can * be done without getting too unfair. */ if (cfs_rq->skip == se) { - struct sched_entity *second = __pick_next_entity(se); + struct sched_entity *second; + + if (se == curr) { + second = __pick_first_entity(cfs_rq); + } else { + second = __pick_next_entity(se); + if (!second || (curr && entity_before(curr, second))) + second = curr; + } + if (second && wakeup_preempt_entity(second, left) < 1) se = second; } @@ -3985,7 +3133,7 @@ static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) return se; } -static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq); +static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) { @@ -4031,7 +3179,7 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) * validating it and just reschedule. */ if (queued) { - resched_task(rq_of(cfs_rq)->curr); + resched_curr(rq_of(cfs_rq)); return; } /* @@ -4094,26 +3242,6 @@ static inline u64 sched_cfs_bandwidth_slice(void) return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; } -#ifdef VENDOR_EDIT -static unsigned int count_currnet_cfs_h_tasks(struct cfs_bandwidth *cfs_b) -{ - struct cfs_rq *cfs_rq; - unsigned int task_count = 0; - - rcu_read_lock(); - list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, - throttled_list) { - task_count += cfs_rq->h_nr_running; - } - list_for_each_entry_rcu(cfs_rq, &cfs_b->unthrottled_cfs_rq, - unthrottled_list) { - task_count += cfs_rq->h_nr_running; - } - rcu_read_unlock(); - return task_count?:1; -} -#endif - /* * Replenish runtime according to assigned quota and update expiration time. * We use sched_clock_cpu directly instead of rq->clock to avoid adding @@ -4124,22 +3252,12 @@ static unsigned int count_currnet_cfs_h_tasks(struct cfs_bandwidth *cfs_b) void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) { u64 now; -#ifdef VENDOR_EDIT - unsigned int nr_h_tasks; -#endif if (cfs_b->quota == RUNTIME_INF) return; now = sched_clock_cpu(smp_processor_id()); -#ifdef VENDOR_EDIT - nr_h_tasks = count_currnet_cfs_h_tasks(cfs_b); - if (cfs_b->quota_per_task != 0 && nr_h_tasks != 0) - cfs_b->runtime = min(cfs_b->quota, cfs_b->quota_per_task * nr_h_tasks); - else -#endif - cfs_b->runtime = cfs_b->quota; - + cfs_b->runtime = cfs_b->quota; cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period); } @@ -4154,7 +3272,7 @@ static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq) if (unlikely(cfs_rq->throttle_count)) return cfs_rq->throttled_clock_task; - return rq_of(cfs_rq)->clock_task - cfs_rq->throttled_clock_task_time; + return rq_clock_task(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; } /* returns 0 on failure to allocate runtime */ @@ -4210,10 +3328,9 @@ static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) static void expire_cfs_rq_runtime(struct cfs_rq *cfs_rq) { struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); - struct rq *rq = rq_of(cfs_rq); /* if the deadline is ahead of our clock, nothing to do */ - if (likely((s64)(rq->clock - cfs_rq->runtime_expires) < 0)) + if (likely((s64)(rq_clock(rq_of(cfs_rq)) - cfs_rq->runtime_expires) < 0)) return; if (cfs_rq->runtime_remaining < 0) @@ -4239,11 +3356,7 @@ static void expire_cfs_rq_runtime(struct cfs_rq *cfs_rq) } } -#ifdef VENDOR_EDIT -static atomic_t skip_throttle = ATOMIC_INIT(0); -#endif -static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, - unsigned long delta_exec) +static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) { /* dock delta_exec before expiring quota (as it could span periods) */ cfs_rq->runtime_remaining -= delta_exec; @@ -4256,17 +3369,12 @@ static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, * if we're unable to extend our runtime we resched so that the active * hierarchy can be throttled */ - -#ifdef VENDOR_EDIT - if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr) && !atomic_read(&skip_throttle)) -#else if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) -#endif - resched_task(rq_of(cfs_rq)->curr); + resched_curr(rq_of(cfs_rq)); } static __always_inline -void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, unsigned long delta_exec) +void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) { if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) return; @@ -4279,33 +3387,6 @@ static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) return cfs_bandwidth_used() && cfs_rq->throttled; } -/* - * Check if task is part of a hierarchy where some cfs_rq does not have any - * runtime left. - * - * We can't rely on throttled_hierarchy() to do this test, as - * cfs_rq->throttle_count will not be updated yet when this function is called - * from scheduler_tick() - */ -static int task_will_be_throttled(struct task_struct *p) -{ - struct sched_entity *se = &p->se; - struct cfs_rq *cfs_rq; - - if (!cfs_bandwidth_used()) - return 0; - - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - if (!cfs_rq->runtime_enabled) - continue; - if (cfs_rq->runtime_remaining <= 0) - return 1; - } - - return 0; -} - /* check whether cfs_rq, or any parent, is throttled */ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) { @@ -4339,7 +3420,7 @@ static int tg_unthrottle_up(struct task_group *tg, void *data) #ifdef CONFIG_SMP if (!cfs_rq->throttle_count) { /* adjust cfs_rq_clock_task() */ - cfs_rq->throttled_clock_task_time += rq->clock_task - + cfs_rq->throttled_clock_task_time += rq_clock_task(rq) - cfs_rq->throttled_clock_task; } #endif @@ -4354,16 +3435,12 @@ static int tg_throttle_down(struct task_group *tg, void *data) /* group is entering throttled state, stop time */ if (!cfs_rq->throttle_count) - cfs_rq->throttled_clock_task = rq->clock_task; + cfs_rq->throttled_clock_task = rq_clock_task(rq); cfs_rq->throttle_count++; return 0; } -#ifdef VENDOR_EDIT -static atomic_t throttle_ongoing = ATOMIC_INIT(0); -static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b); -#endif static void throttle_cfs_rq(struct cfs_rq *cfs_rq) { struct rq *rq = rq_of(cfs_rq); @@ -4371,15 +3448,6 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) struct sched_entity *se; long task_delta, dequeue = 1; -#ifdef VENDOR_EDIT - atomic_inc(&throttle_ongoing); - if (atomic_read(&skip_throttle)) { - atomic_dec(&throttle_ongoing); - pr_debug("SMSG: critical tasks are working, skip throttle\n"); - return; - } -#endif - se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; /* freeze hierarchy runnable averages while throttled */ @@ -4397,44 +3465,25 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) if (dequeue) dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP); qcfs_rq->h_nr_running -= task_delta; - dec_throttled_cfs_rq_hmp_stats(&qcfs_rq->hmp_stats, cfs_rq); if (qcfs_rq->load.weight) dequeue = 0; } - if (!se) { - sched_update_nr_prod(cpu_of(rq), task_delta, false); - rq->nr_running -= task_delta; - dec_throttled_cfs_rq_hmp_stats(&rq->hmp_stats, cfs_rq); - } + if (!se) + sub_nr_running(rq, task_delta); cfs_rq->throttled = 1; - cfs_rq->throttled_clock = rq->clock; + cfs_rq->throttled_clock = rq_clock(rq); raw_spin_lock(&cfs_b->lock); /* * Add to the _head_ of the list, so that an already-started * distribute_cfs_runtime will not see us */ list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); -#ifdef VENDOR_EDIT - list_del_rcu(&cfs_rq->unthrottled_list); -#endif if (!cfs_b->timer_active) __start_cfs_bandwidth(cfs_b, false); - if (atomic_read(&skip_throttle)) - start_cfs_slack_bandwidth(cfs_b); raw_spin_unlock(&cfs_b->lock); -#ifdef VENDOR_EDIT - atomic_dec(&throttle_ongoing); -#endif - - /* Log effect on hmp stats after throttling */ - trace_sched_cpu_load(rq, idle_cpu(cpu_of(rq)), - mostly_idle_cpu(cpu_of(rq)), - sched_irqload(cpu_of(rq)), - power_cost_at_freq(cpu_of(rq), 0), - cpu_temp(cpu_of(rq))); } void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) @@ -4444,20 +3493,18 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) struct sched_entity *se; int enqueue = 1; long task_delta; - struct cfs_rq *tcfs_rq = cfs_rq; - se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; + se = cfs_rq->tg->se[cpu_of(rq)]; cfs_rq->throttled = 0; + + update_rq_clock(rq); + raw_spin_lock(&cfs_b->lock); - cfs_b->throttled_time += rq->clock - cfs_rq->throttled_clock; + cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; list_del_rcu(&cfs_rq->throttled_list); -#ifdef VENDOR_EDIT - list_add_rcu(&cfs_rq->unthrottled_list, &cfs_b->unthrottled_cfs_rq); -#endif raw_spin_unlock(&cfs_b->lock); - update_rq_clock(rq); /* update hierarchical throttle state */ walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); @@ -4473,28 +3520,17 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) if (enqueue) enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP); cfs_rq->h_nr_running += task_delta; - inc_throttled_cfs_rq_hmp_stats(&cfs_rq->hmp_stats, tcfs_rq); if (cfs_rq_throttled(cfs_rq)) break; } - if (!se) { - sched_update_nr_prod(cpu_of(rq), task_delta, true); - rq->nr_running += task_delta; - inc_throttled_cfs_rq_hmp_stats(&rq->hmp_stats, tcfs_rq); - } + if (!se) + add_nr_running(rq, task_delta); /* determine whether we need to wake up potentially idle cpu */ if (rq->curr == rq->idle && rq->cfs.nr_running) - resched_task(rq->curr); - - /* Log effect on hmp stats after un-throttling */ - trace_sched_cpu_load(rq, idle_cpu(cpu_of(rq)), - mostly_idle_cpu(cpu_of(rq)), - sched_irqload(cpu_of(rq)), - power_cost_at_freq(cpu_of(rq), 0), - cpu_temp(cpu_of(rq))); + resched_curr(rq); } static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, @@ -4521,10 +3557,6 @@ static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, cfs_rq->runtime_remaining += runtime; cfs_rq->runtime_expires = expires; - /* when skip_throttle, refill little time to unthrottle all cfs_rq */ - if (atomic_read(&skip_throttle) && cfs_rq->runtime_remaining <= 0) - cfs_rq->runtime_remaining = 1; - /* we check whether we're throttled above */ if (cfs_rq->runtime_remaining > 0) unthrottle_cfs_rq(cfs_rq); @@ -4532,8 +3564,7 @@ static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, next: raw_spin_unlock(&rq->lock); - /* when skip_throttle, need to bring throttle cfs_rq back asap */ - if (!remaining && !atomic_read(&skip_throttle)) + if (!remaining) break; } rcu_read_unlock(); @@ -4563,11 +3594,7 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) * idle depends on !throttled (for the case of a large deficit), and if * we're going inactive then everything else can be deferred */ -#ifdef VENDOR_EDIT - if (cfs_b->idle && !throttled && !atomic_read(&skip_throttle)) -#else if (cfs_b->idle && !throttled) -#endif goto out_deactivate; /* @@ -4597,7 +3624,7 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) * in us over-using our runtime if it is all used during this loop, but * only by limited amounts in that extreme case. */ - while (throttled && (cfs_b->runtime > 0 || atomic_read(&skip_throttle))) { + while (throttled && cfs_b->runtime > 0) { runtime = cfs_b->runtime; raw_spin_unlock(&cfs_b->lock); /* we can't nest cfs_b->lock while distributing bandwidth */ @@ -4726,7 +3753,7 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) expires = cfs_b->runtime_expires; raw_spin_unlock(&cfs_b->lock); - if (!runtime && !atomic_read(&skip_throttle)) + if (!runtime) return; runtime = distribute_cfs_runtime(cfs_b, runtime, expires); @@ -4737,38 +3764,6 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) raw_spin_unlock(&cfs_b->lock); } -#ifdef VENDOR_EDIT -void skip_cfs_throttle(int skip) -{ - struct cfs_bandwidth *cfs_b = &task_group(current)->cfs_bandwidth; - if (skip) { - atomic_inc(&skip_throttle); - while (atomic_read(&throttle_ongoing) && cfs_b->quota != RUNTIME_INF) - schedule(); - } else - atomic_dec(&skip_throttle); - - pr_debug_ratelimited("SMSG: current skip_throttle value=%d\n", atomic_read(&skip_throttle)); -} -#endif - -#ifdef VENDOR_EDIT -static inline void enter_cfs_throttling(atomic_t *in_progress) -{ - atomic_inc(in_progress); -} - -static inline void exit_cfs_throttling(atomic_t *in_progress) -{ - atomic_dec(in_progress); -} - -static inline bool is_throttling_ongoing(atomic_t *in_progress) -{ - return atomic_read(in_progress) == 0 ? false:true; -} -#endif - /* * When a group wakes up we want to make sure that its quota is not already * expired/exceeded, otherwise it may be allowed to steal additional ticks of @@ -4776,81 +3771,43 @@ static inline bool is_throttling_ongoing(atomic_t *in_progress) */ static void check_enqueue_throttle(struct cfs_rq *cfs_rq) { -#ifdef VENDOR_EDIT - atomic_t *tip = &cfs_rq->throttling_in_progress; -#endif if (!cfs_bandwidth_used()) return; -#ifdef VENDOR_EDIT - enter_cfs_throttling(tip); -#endif - /* an active group must be handled by the update_curr()->put() path */ - if (!cfs_rq->runtime_enabled || cfs_rq->curr) { -#ifdef VENDOR_EDIT - exit_cfs_throttling(tip); -#endif + if (!cfs_rq->runtime_enabled || cfs_rq->curr) return; - } /* ensure the group is not already throttled */ - if (cfs_rq_throttled(cfs_rq)) { -#ifdef VENDOR_EDIT - exit_cfs_throttling(tip); -#endif + if (cfs_rq_throttled(cfs_rq)) return; - } /* update runtime allocation */ account_cfs_rq_runtime(cfs_rq, 0); if (cfs_rq->runtime_remaining <= 0) throttle_cfs_rq(cfs_rq); -#ifdef VENDOR_EDIT - exit_cfs_throttling(tip); -#endif } /* conditionally throttle active cfs_rq's from put_prev_entity() */ -static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq) +static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { -#ifdef VENDOR_EDIT - atomic_t *tip = &cfs_rq->throttling_in_progress; -#endif if (!cfs_bandwidth_used()) - return; + return false; -#ifdef VENDOR_EDIT - enter_cfs_throttling(tip); -#endif - if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) { -#ifdef VENDOR_EDIT - exit_cfs_throttling(tip); -#endif - return; - } + if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) + return false; /* * it's possible for a throttled entity to be forced into a running * state (e.g. set_curr_task), in this case we're finished. */ - if (cfs_rq_throttled(cfs_rq)) { -#ifdef VENDOR_EDIT - exit_cfs_throttling(tip); -#endif - return; - } + if (cfs_rq_throttled(cfs_rq)) + return true; throttle_cfs_rq(cfs_rq); -#ifdef VENDOR_EDIT - exit_cfs_throttling(tip); -#endif + return true; } -static inline u64 default_cfs_period(void); -static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun); -static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b); - static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) { struct cfs_bandwidth *cfs_b = @@ -4891,9 +3848,6 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) cfs_b->period = ns_to_ktime(default_cfs_period()); INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); -#ifdef VENDOR_EDIT - INIT_LIST_HEAD(&cfs_b->unthrottled_cfs_rq); -#endif hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); cfs_b->period_timer.function = sched_cfs_period_timer; hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); @@ -4903,11 +3857,7 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) { cfs_rq->runtime_enabled = 0; -#ifdef VENDOR_EDIT - INIT_LIST_HEAD(&cfs_rq->unthrottled_list); -#endif INIT_LIST_HEAD(&cfs_rq->throttled_list); - init_cfs_rq_hmp_stats(cfs_rq); } /* requires cfs_b->lock, may release to reprogram timer */ @@ -4943,36 +3893,21 @@ static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) static void __maybe_unused update_runtime_enabled(struct rq *rq) { struct cfs_rq *cfs_rq; -#ifdef VENDOR_EDIT - struct task_group *tg; - rcu_read_lock(); - for_each_cfs_rq(cfs_rq, tg, cpu_of(rq)) { -#else for_each_leaf_cfs_rq(rq, cfs_rq) { -#endif struct cfs_bandwidth *cfs_b = &cfs_rq->tg->cfs_bandwidth; raw_spin_lock(&cfs_b->lock); cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; raw_spin_unlock(&cfs_b->lock); } -#ifdef VENDOR_EDIT - rcu_read_unlock(); -#endif } -static void unthrottle_offline_cfs_rqs(struct rq *rq) +static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) { struct cfs_rq *cfs_rq; -#ifdef VENDOR_EDIT - struct task_group *tg; - rcu_read_lock(); - for_each_cfs_rq(cfs_rq, tg, cpu_of(rq)) { -#else for_each_leaf_cfs_rq(rq, cfs_rq) { -#endif if (!cfs_rq->runtime_enabled) continue; @@ -4986,28 +3921,20 @@ static void unthrottle_offline_cfs_rqs(struct rq *rq) * in take_cpu_down(), so we prevent new cfs throttling here. */ cfs_rq->runtime_enabled = 0; -#ifdef VENDOR_EDIT - while (is_throttling_ongoing(&cfs_rq->throttling_in_progress)) - ; -#endif if (cfs_rq_throttled(cfs_rq)) unthrottle_cfs_rq(cfs_rq); } -#ifdef VENDOR_EDIT - rcu_read_unlock(); -#endif } #else /* CONFIG_CFS_BANDWIDTH */ static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq) { - return rq_of(cfs_rq)->clock_task; + return rq_clock_task(rq_of(cfs_rq)); } -static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, - unsigned long delta_exec) {} -static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} +static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} +static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; } static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} @@ -5055,31 +3982,24 @@ static void hrtick_start_fair(struct rq *rq, struct task_struct *p) WARN_ON(task_rq(p) != rq); - if (rq->cfs.h_nr_running > 1) { + if (cfs_rq->nr_running > 1) { u64 slice = sched_slice(cfs_rq, se); u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; s64 delta = slice - ran; if (delta < 0) { if (rq->curr == p) - resched_task(p); + resched_curr(rq); return; } - - /* - * Don't schedule slices shorter than 10000ns, that just - * doesn't make sense. Rely on vruntime for fairness. - */ - if (rq->curr != p) - delta = max_t(s64, 10000LL, delta); - hrtick_start(rq, delta); } } /* * called from enqueue/dequeue and updates the hrtick when the - * current task is from our class. + * current task is from our class and nr_running is low enough + * to matter. */ static void hrtick_update(struct rq *rq) { @@ -5088,7 +4008,8 @@ static void hrtick_update(struct rq *rq) if (!hrtick_enabled(rq) || curr->sched_class != &fair_sched_class) return; - hrtick_start_fair(rq, curr); + if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency) + hrtick_start_fair(rq, curr); } #else /* !CONFIG_SCHED_HRTICK */ static inline void @@ -5101,6 +4022,13 @@ static inline void hrtick_update(struct rq *rq) } #endif +unsigned int __read_mostly sysctl_sched_capacity_margin = 1280; /* ~20% margin */ + +static bool cpu_overutilized(int cpu); +static unsigned long get_cpu_usage(int cpu); +static inline unsigned long get_boosted_cpu_usage(int cpu); +struct static_key __sched_energy_freq __read_mostly = STATIC_KEY_INIT_FALSE; + /* * The enqueue_task method is called before nr_running is * increased. Here we update the fair scheduling stats and @@ -5111,6 +4039,8 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) { struct cfs_rq *cfs_rq; struct sched_entity *se = &p->se; + int task_new = flags & ENQUEUE_WAKEUP_NEW; + int task_wakeup = flags & ENQUEUE_WAKEUP; for_each_sched_entity(se) { if (se->on_rq) @@ -5127,7 +4057,6 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (cfs_rq_throttled(cfs_rq)) break; cfs_rq->h_nr_running++; - inc_cfs_rq_hmp_stats(cfs_rq, p, 1); flags = ENQUEUE_WAKEUP; } @@ -5135,7 +4064,6 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); cfs_rq->h_nr_running++; - inc_cfs_rq_hmp_stats(cfs_rq, p, 1); if (cfs_rq_throttled(cfs_rq)) break; @@ -5144,10 +4072,33 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) update_entity_load_avg(se, 1); } - if (!se) { - update_rq_runnable_avg(rq, rq->nr_running); - inc_nr_running(rq); - inc_rq_hmp_stats(rq, p, 1); + if (!se) { + update_rq_runnable_avg(rq, rq->nr_running); + add_nr_running(rq, 1); + if (!task_new && !rq->rd->overutilized && + cpu_overutilized(rq->cpu)) + rq->rd->overutilized = true; + + schedtune_enqueue_task(p, cpu_of(rq)); + + /* + * We want to trigger a freq switch request only for tasks that + * are waking up; this is because we get here also during + * load balancing, but in these cases it seems wise to trigger + * as single request after load balancing is done. + * + * Also, we add a margin (same ~20% used for the tipping point) + * to our request to provide some head room if p's utilization + * further increases. + */ + if (sched_energy_freq() && (task_new || task_wakeup)) { + unsigned long req_cap = + get_boosted_cpu_usage(cpu_of(rq)); + + req_cap = req_cap * sysctl_sched_capacity_margin + >> SCHED_CAPACITY_SHIFT; + cpufreq_sched_set_cap(cpu_of(rq), req_cap); + } } hrtick_update(rq); } @@ -5178,7 +4129,6 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (cfs_rq_throttled(cfs_rq)) break; cfs_rq->h_nr_running--; - dec_cfs_rq_hmp_stats(cfs_rq, p, 1); /* Don't dequeue parent if it has other entities besides us */ if (cfs_rq->load.weight) { @@ -5199,7 +4149,6 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); cfs_rq->h_nr_running--; - dec_cfs_rq_hmp_stats(cfs_rq, p, 1); if (cfs_rq_throttled(cfs_rq)) break; @@ -5209,18 +4158,225 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) } if (!se) { - dec_nr_running(rq); + sub_nr_running(rq, 1); update_rq_runnable_avg(rq, 1); - dec_rq_hmp_stats(rq, p, 1); + + schedtune_dequeue_task(p, cpu_of(rq)); + + /* + * We want to trigger a freq switch request only for tasks that + * are going to sleep; this is because we get here also during + * load balancing, but in these cases it seems wise to trigger + * as single request after load balancing is done. + * + * Also, we add a margin (same ~20% used for the tipping point) + * to our request to provide some head room if p's utilization + * further increases. + */ + if (sched_energy_freq() && task_sleep) { + unsigned long req_cap = + get_boosted_cpu_usage(cpu_of(rq)); + + if (rq->cfs.nr_running) { + req_cap = req_cap * sysctl_sched_capacity_margin + >> SCHED_CAPACITY_SHIFT; + cpufreq_sched_set_cap(cpu_of(rq), req_cap); + } else { + cpufreq_sched_reset_cap(cpu_of(rq)); + } + } } hrtick_update(rq); } #ifdef CONFIG_SMP + +/* + * per rq 'load' arrray crap; XXX kill this. + */ + +/* + * The exact cpuload at various idx values, calculated at every tick would be + * load = (2^idx - 1) / 2^idx * load + 1 / 2^idx * cur_load + * + * If a cpu misses updates for n-1 ticks (as it was idle) and update gets called + * on nth tick when cpu may be busy, then we have: + * load = ((2^idx - 1) / 2^idx)^(n-1) * load + * load = (2^idx - 1) / 2^idx) * load + 1 / 2^idx * cur_load + * + * decay_load_missed() below does efficient calculation of + * load = ((2^idx - 1) / 2^idx)^(n-1) * load + * avoiding 0..n-1 loop doing load = ((2^idx - 1) / 2^idx) * load + * + * The calculation is approximated on a 128 point scale. + * degrade_zero_ticks is the number of ticks after which load at any + * particular idx is approximated to be zero. + * degrade_factor is a precomputed table, a row for each load idx. + * Each column corresponds to degradation factor for a power of two ticks, + * based on 128 point scale. + * Example: + * row 2, col 3 (=12) says that the degradation at load idx 2 after + * 8 ticks is 12/128 (which is an approximation of exact factor 3^8/4^8). + * + * With this power of 2 load factors, we can degrade the load n times + * by looking at 1 bits in n and doing as many mult/shift instead of + * n mult/shifts needed by the exact degradation. + */ +#define DEGRADE_SHIFT 7 +static const unsigned char + degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128}; +static const unsigned char + degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = { + {0, 0, 0, 0, 0, 0, 0, 0}, + {64, 32, 8, 0, 0, 0, 0, 0}, + {96, 72, 40, 12, 1, 0, 0}, + {112, 98, 75, 43, 15, 1, 0}, + {120, 112, 98, 76, 45, 16, 2} }; + +/* + * Update cpu_load for any missed ticks, due to tickless idle. The backlog + * would be when CPU is idle and so we just decay the old load without + * adding any new load. + */ +static unsigned long +decay_load_missed(unsigned long load, unsigned long missed_updates, int idx) +{ + int j = 0; + + if (!missed_updates) + return load; + + if (missed_updates >= degrade_zero_ticks[idx]) + return 0; + + if (idx == 1) + return load >> missed_updates; + + while (missed_updates) { + if (missed_updates % 2) + load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT; + + missed_updates >>= 1; + j++; + } + return load; +} + +/* + * Update rq->cpu_load[] statistics. This function is usually called every + * scheduler tick (TICK_NSEC). With tickless idle this will not be called + * every tick. We fix it up based on jiffies. + */ +static void __update_cpu_load(struct rq *this_rq, unsigned long this_load, + unsigned long pending_updates) +{ + int i, scale; + + this_rq->nr_load_updates++; + + /* Update our load: */ + this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */ + for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) { + unsigned long old_load, new_load; + + /* scale is effectively 1 << i now, and >> i divides by scale */ + + old_load = this_rq->cpu_load[i]; + old_load = decay_load_missed(old_load, pending_updates - 1, i); + new_load = this_load; + /* + * Round up the averaging division if load is increasing. This + * prevents us from getting stuck on 9 if the load is 10, for + * example. + */ + if (new_load > old_load) + new_load += scale - 1; + + this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i; + } + + sched_avg_update(this_rq); +} + +#ifdef CONFIG_NO_HZ_COMMON +/* + * There is no sane way to deal with nohz on smp when using jiffies because the + * cpu doing the jiffies update might drift wrt the cpu doing the jiffy reading + * causing off-by-one errors in observed deltas; {0,2} instead of {1,1}. + * + * Therefore we cannot use the delta approach from the regular tick since that + * would seriously skew the load calculation. However we'll make do for those + * updates happening while idle (nohz_idle_balance) or coming out of idle + * (tick_nohz_idle_exit). + * + * This means we might still be one tick off for nohz periods. + */ + +/* + * Called from nohz_idle_balance() to update the load ratings before doing the + * idle balance. + */ +static void update_idle_cpu_load(struct rq *this_rq) +{ + unsigned long curr_jiffies = ACCESS_ONCE(jiffies); + unsigned long load = this_rq->cfs.runnable_load_avg; + unsigned long pending_updates; + + /* + * bail if there's load or we're actually up-to-date. + */ + if (load || curr_jiffies == this_rq->last_load_update_tick) + return; + + pending_updates = curr_jiffies - this_rq->last_load_update_tick; + this_rq->last_load_update_tick = curr_jiffies; + + __update_cpu_load(this_rq, load, pending_updates); +} + +/* + * Called from tick_nohz_idle_exit() -- try and fix up the ticks we missed. + */ +void update_cpu_load_nohz(void) +{ + struct rq *this_rq = this_rq(); + unsigned long curr_jiffies = ACCESS_ONCE(jiffies); + unsigned long pending_updates; + + if (curr_jiffies == this_rq->last_load_update_tick) + return; + + raw_spin_lock(&this_rq->lock); + pending_updates = curr_jiffies - this_rq->last_load_update_tick; + if (pending_updates) { + this_rq->last_load_update_tick = curr_jiffies; + /* + * We were idle, this means load 0, the current load might be + * !0 due to remote wakeups and the sort. + */ + __update_cpu_load(this_rq, 0, pending_updates); + } + raw_spin_unlock(&this_rq->lock); +} +#endif /* CONFIG_NO_HZ */ + +/* + * Called from scheduler_tick() + */ +void update_cpu_load_active(struct rq *this_rq) +{ + unsigned long load = this_rq->cfs.runnable_load_avg; + /* + * See the mess around update_idle_cpu_load() / update_cpu_load_nohz(). + */ + this_rq->last_load_update_tick = jiffies; + __update_cpu_load(this_rq, load, 1); +} + /* Used instead of source_load when we know the type == 0 */ static unsigned long weighted_cpuload(const int cpu) { - return cpu_rq(cpu)->load.weight; + return cpu_rq(cpu)->cfs.runnable_load_avg; } /* @@ -5256,22 +4412,45 @@ static unsigned long target_load(int cpu, int type) return max(rq->cpu_load[type-1], total); } -static unsigned long power_of(int cpu) +static unsigned long capacity_of(int cpu) +{ + return cpu_rq(cpu)->cpu_capacity; +} + +unsigned long capacity_orig_of(int cpu) { - return cpu_rq(cpu)->cpu_power; + return cpu_rq(cpu)->cpu_capacity_orig; } static unsigned long cpu_avg_load_per_task(int cpu) { struct rq *rq = cpu_rq(cpu); - unsigned long nr_running = ACCESS_ONCE(rq->nr_running); + unsigned long nr_running = ACCESS_ONCE(rq->cfs.h_nr_running); + unsigned long load_avg = rq->cfs.runnable_load_avg; if (nr_running) - return rq->load.weight / nr_running; + return load_avg / nr_running; return 0; } +static void record_wakee(struct task_struct *p) +{ + /* + * Rough decay (wiping) for cost saving, don't worry + * about the boundary, really active task won't care + * about the loss. + */ + if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { + current->wakee_flips >>= 1; + current->wakee_flip_decay_ts = jiffies; + } + + if (current->last_wakee != p) { + current->last_wakee = p; + current->wakee_flips++; + } +} static void task_waking_fair(struct task_struct *p) { @@ -5292,6 +4471,7 @@ static void task_waking_fair(struct task_struct *p) #endif se->vruntime -= min_vruntime; + record_wakee(p); } #ifdef CONFIG_FAIR_GROUP_SCHED @@ -5400,25 +4580,467 @@ static long effective_load(struct task_group *tg, int cpu, long wl, long wg) return wl; } + +#else + +static long effective_load(struct task_group *tg, int cpu, long wl, long wg) +{ + return wl; +} + +#endif + +/* + * Returns the current capacity of cpu after applying both + * cpu and freq scaling. + */ +unsigned long capacity_curr_of(int cpu) +{ + return cpu_rq(cpu)->cpu_capacity_orig * + arch_scale_freq_capacity(NULL, cpu) + >> SCHED_CAPACITY_SHIFT; +} + +/* + * get_cpu_usage returns the amount of capacity of a CPU that is used by CFS + * tasks. The unit of the return value must be the one of capacity so we can + * compare the usage with the capacity of the CPU that is available for CFS + * task (ie cpu_capacity). + * + * cfs.utilization_load_avg is the sum of running time of runnable tasks on a + * CPU. It represents the amount of utilization of a CPU in the range + * [0..capacity_orig] where capacity_orig is the cpu_capacity available at the + * highest frequency (arch_scale_freq_capacity()). The usage of a CPU converges + * towards a sum equal to or less than the current capacity (capacity_curr <= + * capacity_orig) of the CPU because it is the running time on this CPU scaled + * by capacity_curr. Nevertheless, cfs.utilization_load_avg can be higher than + * capacity_curr or even higher than capacity_orig because of unfortunate + * rounding in avg_period and running_load_avg or just after migrating tasks + * (and new task wakeups) until the average stabilizes with the new running + * time. We need to check that the usage stays into the range + * [0..capacity_orig] and cap if necessary. Without capping the usage, a group + * could be seen as overloaded (CPU0 usage at 121% + CPU1 usage at 80%) whereas + * CPU1 has 20% of available capacity. We allow usage to overshoot + * capacity_curr (but not capacity_orig) as it useful for predicting the + * capacity required after task migrations (scheduler-driven DVFS). + */ +static unsigned long __get_cpu_usage(int cpu, int delta) +{ + int sum; + unsigned long usage = cpu_rq(cpu)->cfs.utilization_load_avg; + unsigned long blocked = cpu_rq(cpu)->cfs.utilization_blocked_avg; + unsigned long capacity_orig = capacity_orig_of(cpu); + + sum = usage + blocked + delta; + + if (sum < 0) + return 0; + + if (sum >= capacity_orig) + return capacity_orig; + + return sum; +} + +static unsigned long get_cpu_usage(int cpu) +{ + return __get_cpu_usage(cpu, 0); +} + +static inline bool energy_aware(void) +{ + return sched_feat(ENERGY_AWARE); +} + +struct energy_env { + struct sched_group *sg_top; + struct sched_group *sg_cap; + int cap_idx; + int usage_delta; + int src_cpu; + int dst_cpu; + int energy; + int energy_payoff; + struct task_struct *task; + struct { + int before; + int after; + int diff; + int delta; + } nrg; + struct { + int before; + int after; + int delta; + } cap; +}; + +/* + * __cpu_norm_usage() returns the cpu usage relative to a specific capacity, + * i.e. it's busy ratio, in the range [0..SCHED_LOAD_SCALE] which is useful for + * energy calculations. Using the scale-invariant usage returned by + * get_cpu_usage() and approximating scale-invariant usage by: + * + * usage ~ (curr_freq/max_freq)*1024 * capacity_orig/1024 * running_time/time + * + * the normalized usage can be found using the specific capacity. + * + * capacity = capacity_orig * curr_freq/max_freq + * + * norm_usage = running_time/time ~ usage/capacity + */ +static unsigned long __cpu_norm_usage(int cpu, unsigned long capacity, int delta) +{ + int usage = __get_cpu_usage(cpu, delta); + + if (usage >= capacity) + return SCHED_CAPACITY_SCALE; + + return (usage << SCHED_CAPACITY_SHIFT)/capacity; +} + +static int calc_usage_delta(struct energy_env *eenv, int cpu) +{ + if (cpu == eenv->src_cpu) + return -eenv->usage_delta; + if (cpu == eenv->dst_cpu) + return eenv->usage_delta; + return 0; +} + +static +unsigned long group_max_usage(struct energy_env *eenv) +{ + int i, delta; + unsigned long max_usage = 0; + + for_each_cpu(i, sched_group_cpus(eenv->sg_cap)) { + delta = calc_usage_delta(eenv, i); + max_usage = max(max_usage, __get_cpu_usage(i, delta)); + } + + return max_usage; +} + +/* + * group_norm_usage() returns the approximated group usage relative to it's + * current capacity (busy ratio) in the range [0..SCHED_LOAD_SCALE] for use in + * energy calculations. Since task executions may or may not overlap in time in + * the group the true normalized usage is between max(cpu_norm_usage(i)) and + * sum(cpu_norm_usage(i)) when iterating over all cpus in the group, i. The + * latter is used as the estimate as it leads to a more pessimistic energy + * estimate (more busy). + */ +static unsigned +long group_norm_usage(struct energy_env *eenv, struct sched_group *sg) +{ + int i, delta; + unsigned long usage_sum = 0; + unsigned long capacity = sg->sge->cap_states[eenv->cap_idx].cap; + + for_each_cpu(i, sched_group_cpus(sg)) { + delta = calc_usage_delta(eenv, i); + usage_sum += __cpu_norm_usage(i, capacity, delta); + } + + if (usage_sum > SCHED_CAPACITY_SCALE) + return SCHED_CAPACITY_SCALE; + return usage_sum; +} + +static int find_new_capacity(struct energy_env *eenv, + struct sched_group_energy *sge) +{ + int idx; + unsigned long util = group_max_usage(eenv); + + for (idx = 0; idx < sge->nr_cap_states; idx++) { + if (sge->cap_states[idx].cap >= util) + return idx; + } + + eenv->cap_idx = idx; + + return idx; +} + +static bool cpu_overutilized(int cpu) +{ + return (capacity_of(cpu) * 1024) < + (get_cpu_usage(cpu) * sysctl_sched_capacity_margin); +} + +static int group_idle_state(struct sched_group *sg) +{ + int i, state = INT_MAX; + + /* Find the shallowest idle state in the sched group. */ + for_each_cpu(i, sched_group_cpus(sg)) + state = min(state, idle_get_state_idx(cpu_rq(i))); + + /* Transform system into sched domain idle state. */ + if (sg->sge->nr_idle_states_below > 1) + state -= sg->sge->nr_idle_states_below - 1; + + /* Clamp state to the range of sched domain idle states. */ + return clamp_t(int, state, 0, sg->sge->nr_idle_states - 1); +} + +/* + * sched_group_energy(): Returns absolute energy consumption of cpus belonging + * to the sched_group including shared resources shared only by members of the + * group. Iterates over all cpus in the hierarchy below the sched_group starting + * from the bottom working it's way up before going to the next cpu until all + * cpus are covered at all levels. The current implementation is likely to + * gather the same usage statistics multiple times. This can probably be done in + * a faster but more complex way. + */ +static unsigned int sched_group_energy(struct energy_env *eenv) +{ + struct sched_domain *sd; + int cpu, total_energy = 0; + struct cpumask visit_cpus; + struct sched_group *sg; + + WARN_ON(!eenv->sg_top->sge); + + cpumask_copy(&visit_cpus, sched_group_cpus(eenv->sg_top)); + + while (!cpumask_empty(&visit_cpus)) { + struct sched_group *sg_shared_cap = NULL; + + cpu = cpumask_first(&visit_cpus); + + /* + * Is the group utilization affected by cpus outside this + * sched_group? + */ + sd = highest_flag_domain(cpu, SD_SHARE_CAP_STATES); + if (!sd) + /* + * We most probably raced with hotplug; returning a + * wrong energy estimation is better than entering an + * infinite loop. + */ + break; + + if (sd->parent) + sg_shared_cap = sd->parent->groups; + + for_each_domain(cpu, sd) { + sg = sd->groups; + + /* Has this sched_domain already been visited? */ + if (sd->child && group_first_cpu(sg) != cpu) + break; + + do { + unsigned long group_util; + int sg_busy_energy, sg_idle_energy; + int cap_idx, idle_idx; + + if (sg_shared_cap && sg_shared_cap->group_weight >= sg->group_weight) + eenv->sg_cap = sg_shared_cap; + else + eenv->sg_cap = sg; + + cap_idx = find_new_capacity(eenv, sg->sge); + + if (sg->group_weight == 1) { + /* Remove capacity of src CPU (before task move) */ + if (eenv->usage_delta == 0 && + cpumask_test_cpu(eenv->src_cpu, sched_group_cpus(sg))) { + eenv->cap.before = sg->sge->cap_states[cap_idx].cap; + eenv->cap.delta -= eenv->cap.before; + } + /* Add capacity of dst CPU (after task move) */ + if (eenv->usage_delta != 0 && + cpumask_test_cpu(eenv->dst_cpu, sched_group_cpus(sg))) { + eenv->cap.after = sg->sge->cap_states[cap_idx].cap; + eenv->cap.delta += eenv->cap.after; + } + } + + idle_idx = group_idle_state(sg); + group_util = group_norm_usage(eenv, sg); + sg_busy_energy = (group_util * sg->sge->cap_states[cap_idx].power) + >> SCHED_CAPACITY_SHIFT; + sg_idle_energy = ((SCHED_LOAD_SCALE-group_util) + * sg->sge->idle_states[idle_idx].power) + >> SCHED_CAPACITY_SHIFT; + + total_energy += sg_busy_energy + sg_idle_energy; + + if (!sd->child) + cpumask_xor(&visit_cpus, &visit_cpus, sched_group_cpus(sg)); + + if (cpumask_equal(sched_group_cpus(sg), sched_group_cpus(eenv->sg_top))) + goto next_cpu; + + } while (sg = sg->next, sg != sd->groups); + } +next_cpu: + continue; + } + + eenv->energy = total_energy; + return total_energy; +} + +#ifdef CONFIG_SCHED_TUNE +static int energy_diff_evaluate(struct energy_env *eenv) +{ + unsigned int boost; + int nrg_delta; + + /* Return energy diff when boost margin is 0 */ +#ifdef CONFIG_CGROUP_SCHEDTUNE + boost = schedtune_taskgroup_boost(eenv->task); +#else + boost = get_sysctl_sched_cfs_boost(); +#endif + if (boost == 0) + return eenv->nrg.diff; + + /* Compute normalized energy diff */ + nrg_delta = schedtune_normalize_energy(eenv->nrg.diff); + eenv->nrg.delta = nrg_delta; + +#ifdef CONFIG_CGROUP_SCHEDTUNE + eenv->energy_payoff = schedtune_accept_deltas( + eenv->nrg.delta, + eenv->cap.delta, + eenv->task); #else + eenv->energy_payoff = schedtune_accept_deltas( + eenv->nrg.delta, + eenv->cap.delta); +#endif + + /* + * When SchedTune is enabled, the energy_diff() function will return + * the computed energy payoff value. Since the energy_diff() return + * value is expected to be negative by its callers, this evaluation + * function return a negative value each time the evaluation return a + * positive energy payoff, which is the condition for the acceptance of + * a scheduling decision + */ + return -eenv->energy_payoff; +} +#else /* CONFIG_SCHED_TUNE */ +#define energy_diff_evaluate(eenv) eenv->nrg.diff +#endif + +/* + * energy_diff(): Estimate the energy impact of changing the utilization + * distribution. eenv specifies the change: utilisation amount, source, and + * destination cpu. Source or destination cpu may be -1 in which case the + * utilization is removed from or added to the system (e.g. task wake-up). If + * both are specified, the utilization is migrated. + */ +static int energy_diff(struct energy_env *eenv) +{ + struct sched_domain *sd; + struct sched_group *sg; + int sd_cpu = -1, energy_before = 0, energy_after = 0; + int result; + + struct energy_env eenv_before = { + .usage_delta = 0, + .src_cpu = eenv->src_cpu, + .dst_cpu = eenv->dst_cpu, + .nrg = { 0, 0, 0, 0 }, + .cap = { 0, 0, 0 }, + }; + + if (eenv->src_cpu == eenv->dst_cpu) + return 0; + + sd_cpu = (eenv->src_cpu != -1) ? eenv->src_cpu : eenv->dst_cpu; + sd = rcu_dereference(per_cpu(sd_ea, sd_cpu)); + + if (!sd) + return 0; /* Error */ + + sg = sd->groups; + do { + if (eenv->src_cpu != -1 && cpumask_test_cpu(eenv->src_cpu, + sched_group_cpus(sg))) { + eenv_before.sg_top = eenv->sg_top = sg; + energy_before += sched_group_energy(&eenv_before); + + /* Keep track of SRC cpu (before) capacity */ + eenv->cap.before = eenv_before.cap.before; + eenv->cap.delta = eenv_before.cap.delta; + + energy_after += sched_group_energy(eenv); + + /* src_cpu and dst_cpu may belong to the same group */ + continue; + } + + if (eenv->dst_cpu != -1 && cpumask_test_cpu(eenv->dst_cpu, + sched_group_cpus(sg))) { + eenv_before.sg_top = eenv->sg_top = sg; + energy_before += sched_group_energy(&eenv_before); + energy_after += sched_group_energy(eenv); + } + } while (sg = sg->next, sg != sd->groups); + + eenv->nrg.before = energy_before; + eenv->nrg.after = energy_after; + eenv->nrg.diff = eenv->nrg.after - eenv->nrg.before; + eenv->energy_payoff = 0; + + result = energy_diff_evaluate(eenv); + + trace_sched_energy_diff(eenv->task, + eenv->src_cpu, eenv->dst_cpu, eenv->usage_delta, + eenv->nrg.before, eenv->nrg.after, eenv->nrg.diff, + eenv->cap.before, eenv->cap.after, eenv->cap.delta, + eenv->nrg.delta, eenv->energy_payoff); + + return result; +} + +static int wake_wide(struct task_struct *p) +{ + int factor = this_cpu_read(sd_llc_size); + + /* + * Yeah, it's the switching-frequency, could means many wakee or + * rapidly switch, use factor here will just help to automatically + * adjust the loose-degree, so bigger node will lead to more pull. + */ + if (p->wakee_flips > factor) { + /* + * wakee is somewhat hot, it needs certain amount of cpu + * resource, so if waker is far more hot, prefer to leave + * it alone. + */ + if (current->wakee_flips > (factor * p->wakee_flips)) + return 1; + } -static inline unsigned long effective_load(struct task_group *tg, int cpu, - unsigned long wl, unsigned long wg) -{ - return wl; + return 0; } -#endif - static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync) { s64 this_load, load; + s64 this_eff_load, prev_eff_load; int idx, this_cpu, prev_cpu; - unsigned long tl_per_task; struct task_group *tg; unsigned long weight; int balanced; + /* + * If we wake multiple tasks be careful to not bounce + * ourselves around too much. + */ + if (wake_wide(p)) + return 0; + idx = sd->wake_idx; this_cpu = smp_processor_id(); prev_cpu = task_cpu(p); @@ -5450,63 +5072,230 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync) * Otherwise check if either cpus are near enough in load to allow this * task to be woken on this_cpu. */ - if (this_load > 0) { - s64 this_eff_load, prev_eff_load; + this_eff_load = 100; + this_eff_load *= capacity_of(prev_cpu); - this_eff_load = 100; - this_eff_load *= power_of(prev_cpu); + prev_eff_load = 100 + (sd->imbalance_pct - 100) / 2; + prev_eff_load *= capacity_of(this_cpu); + + if (this_load > 0) { this_eff_load *= this_load + effective_load(tg, this_cpu, weight, weight); - prev_eff_load = 100 + (sd->imbalance_pct - 100) / 2; - prev_eff_load *= power_of(this_cpu); prev_eff_load *= load + effective_load(tg, prev_cpu, 0, weight); + } - balanced = this_eff_load <= prev_eff_load; - } else - balanced = true; + balanced = this_eff_load <= prev_eff_load; + + schedstat_inc(p, se.statistics.nr_wakeups_affine_attempts); + + if (!balanced) + return 0; + + schedstat_inc(sd, ttwu_move_affine); + schedstat_inc(p, se.statistics.nr_wakeups_affine); + + return 1; +} + +static inline unsigned long task_utilization(struct task_struct *p) +{ + return p->se.avg.utilization_avg_contrib; +} + +#ifdef CONFIG_SCHED_TUNE + +static unsigned long +schedtune_margin(unsigned long signal, unsigned long boost) +{ + unsigned long long margin = 0; /* - * If the currently running task will sleep within - * a reasonable amount of time then attract this newly - * woken task: + * Signal proportional compensation (SPC) + * + * The Boost (B) value is used to compute a Maring (M) which is + * proportional to the complement of the original Signal (S): + * M = B * (1024-S) + * The obtained M could be used by the caller to "boost" S. */ - if (sync && balanced) - return 1; + margin = SCHED_LOAD_SCALE - signal; + margin *= boost; - schedstat_inc(p, se.statistics.nr_wakeups_affine_attempts); - tl_per_task = cpu_avg_load_per_task(this_cpu); + /* + * Fast integer division by constant: + * Constant : (C) = 100 + * Precision : 0.1% (P) = 0.1 + * Reference : C * 100 / P (R) = 100000 + * + * Thus: + * Shift bifs : ceil(log(R,2)) (S) = 17 + * Mult const : round(2^S/C) (M) = 1311 + * + * + * */ + margin *= 1311; + margin >>= 17; - if (balanced || - (this_load <= load && - this_load + target_load(prev_cpu, idx) <= tl_per_task)) { - /* - * This domain has SD_WAKE_AFFINE and - * p is cache cold in this domain, and - * there is no bad imbalance. - */ - schedstat_inc(sd, ttwu_move_affine); - schedstat_inc(p, se.statistics.nr_wakeups_affine); + return margin; - return 1; +} + +static unsigned long +schedtune_task_margin(struct task_struct *task) +{ + unsigned int boost; + unsigned long utilization; + unsigned long margin; + +#ifdef CONFIG_CGROUP_SCHEDTUNE + boost = schedtune_taskgroup_boost(task); +#else + boost = get_sysctl_sched_cfs_boost(); +#endif + if (boost == 0) + return 0; + + utilization = task_utilization(task); + margin = schedtune_margin(utilization, boost); + + return margin; +} + +static unsigned long +boosted_task_utilization(struct task_struct *task) +{ + unsigned long utilization; + unsigned long margin = 0; + + utilization = task_utilization(task); + + /* + * Boosting of task utilization is enabled only when the scheduler is + * working in energy-aware mode. + */ + if (!task_rq(task)->rd->overutilized) + margin = schedtune_task_margin(task); + + trace_sched_boost_task(task, utilization, margin); + + utilization += margin; + + return utilization; +} + +#else /* CONFIG_SCHED_TUNE */ + +static unsigned long +boosted_task_utilization(struct task_struct *task) +{ + return task_utilization(task); +} + +#endif /* CONFIG_SCHED_TUNE */ + +static inline bool __task_fits(struct task_struct *p, int cpu, int usage) +{ + unsigned long capacity = capacity_of(cpu); + int orig_usage = usage; + + usage += boosted_task_utilization(p); + + trace_sched_task_fits(p, cpu, orig_usage, usage, task_utilization(p), capacity, sysctl_sched_capacity_margin); + + return (capacity * 1024) > (usage * sysctl_sched_capacity_margin); +} + +static inline bool task_fits_capacity(struct task_struct *p, int cpu) +{ + unsigned long capacity = capacity_of(cpu); + unsigned long max_capacity = cpu_rq(cpu)->rd->max_cpu_capacity; + + if (capacity == max_capacity) + return true; + + if (capacity * sysctl_sched_capacity_margin > max_capacity * 1024) + return true; + + return __task_fits(p, cpu, 0); +} + +static inline bool task_fits_cpu(struct task_struct *p, int cpu) +{ + return __task_fits(p, cpu, get_cpu_usage(cpu)); +} + +#ifdef CONFIG_SCHED_TUNE + +static inline unsigned int +schedtune_cpu_margin(int cpu, unsigned long usage) +{ + unsigned int boost; + unsigned long margin; + +#ifdef CONFIG_CGROUP_SCHEDTUNE + if (usage) + boost = schedtune_cpu_boost(cpu); + else { + boost = 0; + schedtune_idle(cpu); } +#else + boost = get_sysctl_sched_cfs_boost(); +#endif + if (boost == 0) + return 0; + + margin = schedtune_margin(usage, boost); + + return margin; +} + +#else /* CONFIG_SCHED_TUNE */ + +static inline unsigned int +schedtune_cpu_margin(int cpu, unsigned long usage) +{ return 0; } +#endif /* CONFIG_SCHED_TUNE */ + +static inline unsigned long +get_boosted_cpu_usage(int cpu) +{ + unsigned long usage; + unsigned long margin; + + usage = get_cpu_usage(cpu); + margin = schedtune_cpu_margin(cpu, usage); + + trace_sched_boost_cpu(cpu, usage, margin); + + usage += margin; + return usage; +} + /* * find_idlest_group finds and returns the least busy CPU group within the * domain. */ static struct sched_group * find_idlest_group(struct sched_domain *sd, struct task_struct *p, - int this_cpu, int load_idx) + int this_cpu, int sd_flag) { struct sched_group *idlest = NULL, *group = sd->groups; + struct sched_group *fit_group = NULL, *spare_group = NULL; unsigned long min_load = ULONG_MAX, this_load = 0; + unsigned long fit_capacity = ULONG_MAX; + unsigned long max_spare_capacity = sysctl_sched_capacity_margin - SCHED_LOAD_SCALE; + int load_idx = sd->forkexec_idx; int imbalance = 100 + (sd->imbalance_pct-100)/2; + if (sd_flag & SD_BALANCE_WAKE) + load_idx = sd->wake_idx; + do { - unsigned long load, avg_load; + unsigned long load, avg_load, spare_capacity; int local_group; int i; @@ -5529,10 +5318,30 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, load = target_load(i, load_idx); avg_load += load; + + /* + * Look for most energy-efficient group that can fit + * that can fit the task. + */ + if (energy_aware() && capacity_of(i) < fit_capacity && + task_fits_cpu(p, i)) { + fit_capacity = capacity_of(i); + fit_group = group; + } + + /* + * Look for group which has most spare capacity on a + * single cpu. + */ + spare_capacity = capacity_of(i) - get_cpu_usage(i); + if (spare_capacity > max_spare_capacity) { + max_spare_capacity = spare_capacity; + spare_group = group; + } } - /* Adjust by relative CPU power of the group */ - avg_load = (avg_load * SCHED_POWER_SCALE) / group->sgp->power; + /* Adjust by relative CPU capacity of the group */ + avg_load = (avg_load * SCHED_CAPACITY_SCALE) / group->sgc->capacity; if (local_group) { this_load = avg_load; @@ -5542,6 +5351,12 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, } } while (group = group->next, group != sd->groups); + if (fit_group) + return fit_group; + + if (spare_group) + return spare_group; + if (!idlest || 100*this_load < imbalance*min_load) return NULL; return idlest; @@ -5554,20 +5369,54 @@ static int find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) { unsigned long load, min_load = ULONG_MAX; - int idlest = -1; + unsigned int min_exit_latency = UINT_MAX; + u64 latest_idle_timestamp = 0; + int least_loaded_cpu = this_cpu; + int shallowest_idle_cpu = -1; int i; /* Traverse only the allowed CPUs */ for_each_cpu_and(i, sched_group_cpus(group), tsk_cpus_allowed(p)) { - load = weighted_cpuload(i); - - if (load < min_load || (load == min_load && i == this_cpu)) { - min_load = load; - idlest = i; + if (task_fits_cpu(p, i)) { + struct rq *rq = cpu_rq(i); + struct cpuidle_state *idle = idle_get_state(rq); + if (idle && idle->exit_latency < min_exit_latency) { + /* + * We give priority to a CPU whose idle state + * has the smallest exit latency irrespective + * of any idle timestamp. + */ + min_exit_latency = idle->exit_latency; + latest_idle_timestamp = rq->idle_stamp; + shallowest_idle_cpu = i; + } else if (idle_cpu(i) && + (!idle || idle->exit_latency == min_exit_latency) && + rq->idle_stamp > latest_idle_timestamp) { + /* + * If equal or no active idle state, then + * the most recently idled CPU might have + * a warmer cache. + */ + latest_idle_timestamp = rq->idle_stamp; + shallowest_idle_cpu = i; + } else if (shallowest_idle_cpu == -1) { + /* + * If we haven't found an idle CPU yet + * pick a non-idle one that can fit the task as + * fallback. + */ + shallowest_idle_cpu = i; + } + } else { + load = weighted_cpuload(i); + if (load < min_load || (load == min_load && i == this_cpu)) { + min_load = load; + least_loaded_cpu = i; + } } } - return idlest; + return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; } /* @@ -5578,20 +5427,19 @@ static int select_idle_sibling(struct task_struct *p, int target) struct sched_domain *sd; struct sched_group *sg; int i = task_cpu(p); + int best_idle = -1; + int best_idle_cstate = -1; + int best_idle_capacity = INT_MAX; +#if 0 if (idle_cpu(target)) return target; - /* * If the prevous cpu is cache affine and idle, don't be stupid. */ if (i != target && cpus_share_cache(i, target) && idle_cpu(i)) return i; - - if (!sysctl_sched_wake_to_idle && - !(current->flags & PF_WAKE_UP_IDLE) && - !(p->flags & PF_WAKE_UP_IDLE)) - return target; +#endif /* * Otherwise, iterate the domains and find an elegible idle cpu. @@ -5604,54 +5452,218 @@ static int select_idle_sibling(struct task_struct *p, int target) tsk_cpus_allowed(p))) goto next; - for_each_cpu(i, sched_group_cpus(sg)) { - if (i == target || !idle_cpu(i)) + for_each_cpu_and(i, tsk_cpus_allowed(p), sched_group_cpus(sg)) { + struct rq *rq = cpu_rq(i); + int idle_idx = idle_get_state_idx(rq); + unsigned long new_usage = boosted_task_utilization(p); + unsigned long capacity_orig = capacity_orig_of(i); + if (new_usage > capacity_orig || !idle_cpu(i)) goto next; + + if (i == target && new_usage <= capacity_curr_of(target)) + return target; + + if (best_idle < 0 || + (idle_idx < best_idle_cstate && capacity_orig <= best_idle_capacity)) { + best_idle = i; + best_idle_cstate = idle_idx; + best_idle_capacity = capacity_orig; + } } - target = cpumask_first_and(sched_group_cpus(sg), - tsk_cpus_allowed(p)); - goto done; next: sg = sg->next; } while (sg != sd->groups); } -done: + + if (best_idle >= 0) + target = best_idle; + return target; } +static inline int find_best_target(struct task_struct *p) +{ + int i; + int target_cpu = -1; + int target_capacity = 0; + int idle_cpu = -1; + int backup_cpu = -1; + + /* + * Favor 1) busy cpu with most capacity at current OPP + * 2) idle_cpu with capacity at current OPP + * 3) busy cpu with capacity at higher OPP + */ + for_each_cpu(i, tsk_cpus_allowed(p)) { + + int cur_capacity = capacity_curr_of(i); + + /* + * p's blocked utilization is still accounted for on prev_cpu + * so prev_cpu will receive a negative bias due to the double + * accounting. However, the blocked utilization may be zero. + */ + int new_util = get_cpu_usage(i) + boosted_task_utilization(p); + + if (new_util > capacity_orig_of(i)) + continue; + + if (new_util < cur_capacity) { + + if (cpu_rq(i)->nr_running) { + if (target_capacity < cur_capacity) { + /* busy CPU with most capacity at current OPP */ + target_cpu = i; + target_capacity = cur_capacity; + } + } else if (idle_cpu < 0) { + /* first idle CPU with capacity at current OPP */ + idle_cpu = i; + } + } else if (backup_cpu < 0 && cpu_rq(i)->nr_running) { + /* first busy CPU with capacity at higher OPP */ + backup_cpu = i; + } + } + + if (target_cpu < 0) { + target_cpu = idle_cpu >= 0 ? idle_cpu : backup_cpu; + } + + return target_cpu; +} + +static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync) +{ + struct sched_domain *sd; + struct sched_group *sg, *sg_target; + int target_max_cap = INT_MAX; + int target_cpu = -1; + int i; + + if (sysctl_sched_sync_hint_enable && sync) { + int cpu = smp_processor_id(); + cpumask_t search_cpus; + cpumask_and(&search_cpus, tsk_cpus_allowed(p), cpu_online_mask); + if (cpumask_test_cpu(cpu, &search_cpus)) + return cpu; + } + + sd = rcu_dereference(per_cpu(sd_ea, task_cpu(p))); + + if (!sd) + return target; + + sg = sd->groups; + sg_target = sg; + if (sysctl_sched_is_big_little) { + /* + * Find group with sufficient capacity. We only get here if no cpu is + * overutilized. We may end up overutilizing a cpu by adding the task, + * but that should not be any worse than select_idle_sibling(). + * load_balance() should sort it out later as we get above the tipping + * point. + */ + do { + /* Assuming all cpus are the same in group */ + int max_cap_cpu = group_first_cpu(sg); + + /* + * Assume smaller max capacity means more energy-efficient. + * Ideally we should query the energy model for the right + * answer but it easily ends up in an exhaustive search. + */ + if (capacity_of(max_cap_cpu) < target_max_cap && + task_fits_capacity(p, max_cap_cpu)) { + sg_target = sg; + target_max_cap = capacity_of(max_cap_cpu); + } + + } while (sg = sg->next, sg != sd->groups); + + /* Find cpu with sufficient capacity */ + for_each_cpu_and(i, tsk_cpus_allowed(p), sched_group_cpus(sg_target)) { + /* + * p's blocked utilization is still accounted for on prev_cpu + * so prev_cpu will receive a negative bias due the double + * accouting. However, the blocked utilization may be zero. + */ + int new_usage = get_cpu_usage(i) + boosted_task_utilization(p); + + if (new_usage > capacity_orig_of(i)) + continue; + + if (new_usage < capacity_curr_of(i)) { + target_cpu = i; + if (cpu_rq(i)->nr_running) + break; + } + + /* cpu has capacity at higher OPP, keep it as fallback */ + if (target_cpu < 0) + target_cpu = i; + } + } else { + /* + * Find a cpu with sufficient capacity + */ + int tmp_target = find_best_target(p); + if (tmp_target >= 0) target_cpu = tmp_target; + } + + if (target_cpu < 0) + target_cpu = task_cpu(p); + else if (target_cpu != task_cpu(p)) { + struct energy_env eenv = { + .usage_delta = task_utilization(p), + .src_cpu = task_cpu(p), + .dst_cpu = target_cpu, + .task = p, + }; + + /* Not enough spare capacity on previous cpu */ + if (cpu_overutilized(task_cpu(p))) + return target_cpu; + + if (energy_diff(&eenv) >= 0) + return task_cpu(p); + } + + return target_cpu; +} + /* - * sched_balance_self: balance the current task (running on cpu) in domains - * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and - * SD_BALANCE_EXEC. + * select_task_rq_fair: Select target runqueue for the waking task in domains + * that have the 'sd_flag' flag set. In practice, this is SD_BALANCE_WAKE, + * SD_BALANCE_FORK, or SD_BALANCE_EXEC. * - * Balance, ie. select the least loaded group. + * Balances load by selecting the idlest cpu in the idlest group, or under + * certain conditions an idle sibling cpu if the domain has SD_WAKE_AFFINE set. * - * Returns the target CPU number, or the same CPU if no balancing is needed. + * Returns the target cpu number. * * preempt must be disabled. */ static int -select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags) +select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags) { struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL; int cpu = smp_processor_id(); - int prev_cpu = task_cpu(p); int new_cpu = cpu; int want_affine = 0; + int want_sibling = true; int sync = wake_flags & WF_SYNC; if (p->nr_cpus_allowed == 1) return prev_cpu; - if (sched_enable_hmp) - return select_best_cpu(p, prev_cpu, 0, sync); + /* Check if prev_cpu can fit us ignoring its current usage */ + if (energy_aware() && !task_fits_capacity(p, prev_cpu)) + want_sibling = false; - if (sd_flag & SD_BALANCE_WAKE) { - if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) - want_affine = 1; - new_cpu = prev_cpu; - } + if (sd_flag & SD_BALANCE_WAKE && want_sibling) + want_affine = cpumask_test_cpu(cpu, tsk_cpus_allowed(p)); rcu_read_lock(); for_each_domain(cpu, tmp) { @@ -5672,16 +5684,18 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags) sd = tmp; } - if (affine_sd) { - if (cpu != prev_cpu && wake_affine(affine_sd, p, sync)) - prev_cpu = cpu; + if (affine_sd && cpu != prev_cpu && wake_affine(affine_sd, p, sync)) + prev_cpu = cpu; - new_cpu = select_idle_sibling(p, prev_cpu); + if (sd_flag & SD_BALANCE_WAKE && want_sibling) { + if (energy_aware() && !cpu_rq(cpu)->rd->overutilized) + new_cpu = energy_aware_wake_cpu(p, prev_cpu, sync); + else + new_cpu = select_idle_sibling(p, prev_cpu); goto unlock; } while (sd) { - int load_idx = sd->forkexec_idx; struct sched_group *group; int weight; @@ -5690,10 +5704,7 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags) continue; } - if (sd_flag & SD_BALANCE_WAKE) - load_idx = sd->wake_idx; - - group = find_idlest_group(sd, p, cpu, load_idx); + group = find_idlest_group(sd, p, cpu, sd_flag); if (!group) { sd = sd->child; continue; @@ -5724,12 +5735,6 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags) return new_cpu; } -/* - * Load-tracking only depends on SMP, FAIR_GROUP_SCHED dependency below may be - * removed when useful for applications beyond shares distribution (e.g. - * load-balance). - */ -#ifdef CONFIG_FAIR_GROUP_SCHED /* * Called immediately before a task is migrated to a new cpu; task_cpu(p) and * cfs_rq_of(p) references at time of call are still valid and identify the @@ -5750,10 +5755,15 @@ migrate_task_rq_fair(struct task_struct *p, int next_cpu) */ if (se->avg.decay_count) { se->avg.decay_count = -__synchronize_entity_decay(se); - atomic64_add(se->avg.load_avg_contrib, &cfs_rq->removed_load); + atomic_long_add(se->avg.load_avg_contrib, + &cfs_rq->removed_load); + atomic_long_add(se->avg.utilization_avg_contrib, + &cfs_rq->removed_utilization); } + + /* We have migrated, no longer consider this task hot */ + se->exec_start = 0; } -#endif #endif /* CONFIG_SMP */ static unsigned long @@ -5845,7 +5855,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ return; /* - * This is possible from callers such as move_task(), in which we + * This is possible from callers such as attach_tasks(), in which we * unconditionally check_prempt_curr() after an enqueue (which may have * lead to a throttle). This both saves work and prevents false * next-buddy nomination below. @@ -5899,7 +5909,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ return; preempt: - resched_task(curr); + resched_curr(rq); /* * Only set the backward buddy when the current task is still * on the rq. This can happen when a wakeup gets interleaved @@ -5916,26 +5926,133 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ set_last_buddy(se); } -static struct task_struct *pick_next_task_fair(struct rq *rq) +static struct task_struct * +pick_next_task_fair(struct rq *rq, struct task_struct *prev) { - struct task_struct *p; struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *se; + struct task_struct *p; + int new_tasks; +again: +#ifdef CONFIG_FAIR_GROUP_SCHED if (!cfs_rq->nr_running) - return NULL; + goto idle; + + if (prev->sched_class != &fair_sched_class) + goto simple; + + /* + * Because of the set_next_buddy() in dequeue_task_fair() it is rather + * likely that a next task is from the same cgroup as the current. + * + * Therefore attempt to avoid putting and setting the entire cgroup + * hierarchy, only change the part that actually changes. + */ + + do { + struct sched_entity *curr = cfs_rq->curr; + + /* + * Since we got here without doing put_prev_entity() we also + * have to consider cfs_rq->curr. If it is still a runnable + * entity, update_curr() will update its vruntime, otherwise + * forget we've ever seen it. + */ + if (curr) { + if (curr->on_rq) + update_curr(cfs_rq); + else + curr = NULL; + + /* + * This call to check_cfs_rq_runtime() will do the + * throttle and dequeue its entity in the parent(s). + * Therefore the 'simple' nr_running test will indeed + * be correct. + */ + if (unlikely(check_cfs_rq_runtime(cfs_rq))) + goto simple; + } + + se = pick_next_entity(cfs_rq, curr); + cfs_rq = group_cfs_rq(se); + } while (cfs_rq); + + p = task_of(se); + + /* + * Since we haven't yet done put_prev_entity and if the selected task + * is a different task than we started out with, try and touch the + * least amount of cfs_rqs. + */ + if (prev != p) { + struct sched_entity *pse = &prev->se; + + while (!(cfs_rq = is_same_group(se, pse))) { + int se_depth = se->depth; + int pse_depth = pse->depth; + + if (se_depth <= pse_depth) { + put_prev_entity(cfs_rq_of(pse), pse); + pse = parent_entity(pse); + } + if (se_depth >= pse_depth) { + set_next_entity(cfs_rq_of(se), se); + se = parent_entity(se); + } + } + + put_prev_entity(cfs_rq, pse); + set_next_entity(cfs_rq, se); + } + + if (hrtick_enabled(rq)) + hrtick_start_fair(rq, p); + + rq->misfit_task = !task_fits_capacity(p, rq->cpu); + + return p; +simple: + cfs_rq = &rq->cfs; +#endif + + if (!cfs_rq->nr_running) + goto idle; + + put_prev_task(rq, prev); do { - se = pick_next_entity(cfs_rq); + se = pick_next_entity(cfs_rq, NULL); set_next_entity(cfs_rq, se); cfs_rq = group_cfs_rq(se); } while (cfs_rq); p = task_of(se); + if (hrtick_enabled(rq)) hrtick_start_fair(rq, p); - return p; + rq->misfit_task = !task_fits_capacity(p, rq->cpu); + + return p; + +idle: + rq->misfit_task = 0; + + new_tasks = idle_balance(rq); + /* + * Because idle_balance() releases (and re-acquires) rq->lock, it is + * possible for any higher priority task to appear. In that case we + * must re-start the pick_next_entity() loop. + */ + if (new_tasks < 0) + return RETRY_TASK; + + if (new_tasks > 0) + goto again; + + return NULL; } /* @@ -6029,14 +6146,14 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preemp * * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) * - * P_i is the cpu power (or compute capacity) of cpu i, typically it is the + * C_i is the compute capacity of cpu i, typically it is the * fraction of 'recent' time available for SCHED_OTHER task execution. But it * can also include other factors [XXX]. * * To achieve this balance we define a measure of imbalance which follows * directly from (1): * - * imb_i,j = max{ avg(W/P), W_i/P_i } - min{ avg(W/P), W_j/P_j } (4) + * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4) * * We them move tasks around to minimize the imbalance. In the continuous * function space it is obvious this converges, in the discrete case we get @@ -6125,17 +6242,19 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preemp static unsigned long __read_mostly max_load_balance_interval = HZ/10; +enum fbq_type { regular, remote, all }; + +enum group_type { + group_other = 0, + group_misfit_task, + group_imbalanced, + group_overloaded, +}; + #define LBF_ALL_PINNED 0x01 #define LBF_NEED_BREAK 0x02 -#define LBF_SOME_PINNED 0x04 -#define LBF_IGNORE_SMALL_TASKS 0x08 -#define LBF_EA_ACTIVE_BALANCE 0x10 -#define LBF_SCHED_BOOST_ACTIVE_BALANCE 0x20 -#define LBF_BIG_TASK_ACTIVE_BALANCE 0x40 -#define LBF_HMP_ACTIVE_BALANCE (LBF_EA_ACTIVE_BALANCE | \ - LBF_SCHED_BOOST_ACTIVE_BALANCE | \ - LBF_BIG_TASK_ACTIVE_BALANCE) -#define LBF_IGNORE_BIG_TASKS 0x80 +#define LBF_DST_PINNED 0x04 +#define LBF_SOME_PINNED 0x08 struct lb_env { struct sched_domain *sd; @@ -6150,43 +6269,30 @@ struct lb_env { int new_dst_cpu; enum cpu_idle_type idle; long imbalance; + unsigned int src_grp_nr_running; /* The set of CPUs under consideration for load-balancing */ struct cpumask *cpus; - unsigned int busiest_grp_capacity; - unsigned int busiest_nr_running; unsigned int flags; unsigned int loop; unsigned int loop_break; unsigned int loop_max; -}; - -static DEFINE_PER_CPU(bool, dbs_boost_needed); -static DEFINE_PER_CPU(int, dbs_boost_load_moved); -/* - * move_task - move a task from one runqueue to another runqueue. - * Both runqueues must be locked. - */ -static void move_task(struct task_struct *p, struct lb_env *env) -{ - deactivate_task(env->src_rq, p, 0); - set_task_cpu(p, env->dst_cpu); - activate_task(env->dst_rq, p, 0); - check_preempt_curr(env->dst_rq, p, 0); - if (task_notify_on_migrate(p)) - per_cpu(dbs_boost_needed, env->dst_cpu) = true; -} + enum fbq_type fbq_type; + enum group_type busiest_group_type; + struct list_head tasks; +}; /* * Is this task likely cache-hot: */ -static int -task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) +static int task_hot(struct task_struct *p, struct lb_env *env) { s64 delta; + lockdep_assert_held(&env->src_rq->lock); + if (p->sched_class != &fair_sched_class) return 0; @@ -6196,7 +6302,7 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) /* * Buddy candidates are cache hot: */ - if (sched_feat(CACHE_HOT_BUDDY) && this_rq()->nr_running && + if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && (&p->se == cfs_rq_of(&p->se)->next || &p->se == cfs_rq_of(&p->se)->last)) return 1; @@ -6206,11 +6312,98 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) if (sysctl_sched_migration_cost == 0) return 0; - delta = now - p->se.exec_start; + delta = rq_clock_task(env->src_rq) - p->se.exec_start; return delta < (s64)sysctl_sched_migration_cost; } +#ifdef CONFIG_NUMA_BALANCING +/* Returns true if the destination node has incurred more faults */ +static bool migrate_improves_locality(struct task_struct *p, struct lb_env *env) +{ + struct numa_group *numa_group = rcu_dereference(p->numa_group); + int src_nid, dst_nid; + + if (!sched_feat(NUMA_FAVOUR_HIGHER) || !p->numa_faults_memory || + !(env->sd->flags & SD_NUMA)) { + return false; + } + + src_nid = cpu_to_node(env->src_cpu); + dst_nid = cpu_to_node(env->dst_cpu); + + if (src_nid == dst_nid) + return false; + + if (numa_group) { + /* Task is already in the group's interleave set. */ + if (node_isset(src_nid, numa_group->active_nodes)) + return false; + + /* Task is moving into the group's interleave set. */ + if (node_isset(dst_nid, numa_group->active_nodes)) + return true; + + return group_faults(p, dst_nid) > group_faults(p, src_nid); + } + + /* Encourage migration to the preferred node. */ + if (dst_nid == p->numa_preferred_nid) + return true; + + return task_faults(p, dst_nid) > task_faults(p, src_nid); +} + +static bool migrate_degrades_locality(struct task_struct *p, struct lb_env *env) +{ + struct numa_group *numa_group = rcu_dereference(p->numa_group); + int src_nid, dst_nid; + + if (!sched_feat(NUMA) || !sched_feat(NUMA_RESIST_LOWER)) + return false; + + if (!p->numa_faults_memory || !(env->sd->flags & SD_NUMA)) + return false; + + src_nid = cpu_to_node(env->src_cpu); + dst_nid = cpu_to_node(env->dst_cpu); + + if (src_nid == dst_nid) + return false; + + if (numa_group) { + /* Task is moving within/into the group's interleave set. */ + if (node_isset(dst_nid, numa_group->active_nodes)) + return false; + + /* Task is moving out of the group's interleave set. */ + if (node_isset(src_nid, numa_group->active_nodes)) + return true; + + return group_faults(p, dst_nid) < group_faults(p, src_nid); + } + + /* Migrating away from the preferred node is always bad. */ + if (src_nid == p->numa_preferred_nid) + return true; + + return task_faults(p, dst_nid) < task_faults(p, src_nid); +} + +#else +static inline bool migrate_improves_locality(struct task_struct *p, + struct lb_env *env) +{ + return false; +} + +static inline bool migrate_degrades_locality(struct task_struct *p, + struct lb_env *env) +{ + return false; +} +#endif + /* * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? */ @@ -6218,7 +6411,9 @@ static int can_migrate_task(struct task_struct *p, struct lb_env *env) { int tsk_cache_hot = 0; - int twf; + + lockdep_assert_held(&env->src_rq->lock); + /* * We do not migrate tasks that are: * 1) throttled_lb_pair, or @@ -6229,37 +6424,13 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu)) return 0; - if (nr_big_tasks(env->src_rq) && - capacity(env->dst_rq) > capacity(env->src_rq) && - !is_big_task(p)) - return 0; - - if (env->flags & LBF_IGNORE_SMALL_TASKS && is_small_task(p)) - return 0; - - twf = task_will_fit(p, env->dst_cpu); - - /* - * Attempt to not pull tasks that don't fit. We may get lucky and find - * one that actually fits. - */ - if (env->flags & LBF_IGNORE_BIG_TASKS && !twf) - return 0; - - /* - * Group imbalance can sometimes cause work to be pulled across groups - * even though the group could have managed the imbalance on its own. - * Prevent inter-cluster migrations for big tasks when the number of - * tasks is lower than the capacity of the group. - */ - if (!twf && env->busiest_nr_running <= env->busiest_grp_capacity) - return 0; - if (!cpumask_test_cpu(env->dst_cpu, tsk_cpus_allowed(p))) { int cpu; schedstat_inc(p, se.statistics.nr_failed_migrations_affine); + env->flags |= LBF_SOME_PINNED; + /* * Remember if this task can be migrated to any other cpu in * our sched_group. We may want to revisit it if we couldn't @@ -6268,13 +6439,13 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) * Also avoid computing new_dst_cpu if we have already computed * one in current iteration. */ - if (!env->dst_grpmask || (env->flags & LBF_SOME_PINNED)) + if (!env->dst_grpmask || (env->flags & LBF_DST_PINNED)) return 0; /* Prevent to re-select dst_cpu via env's cpus */ for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) { if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) { - env->flags |= LBF_SOME_PINNED; + env->flags |= LBF_DST_PINNED; env->new_dst_cpu = cpu; break; } @@ -6293,19 +6464,20 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) /* * Aggressive migration if: - * 1) task is cache cold, or - * 2) too many balance attempts have failed. + * 1) destination numa is preferred + * 2) task is cache cold, or + * 3) too many balance attempts have failed. */ + tsk_cache_hot = task_hot(p, env); + if (!tsk_cache_hot) + tsk_cache_hot = migrate_degrades_locality(p, env); - tsk_cache_hot = task_hot(p, env->src_rq->clock_task, env->sd); - if (!tsk_cache_hot || - env->sd->nr_balance_failed > env->sd->cache_nice_tries) { - + if (migrate_improves_locality(p, env) || !tsk_cache_hot || + env->sd->nr_balance_failed > env->sd->cache_nice_tries) { if (tsk_cache_hot) { schedstat_inc(env->sd, lb_hot_gained[env->idle]); schedstat_inc(p, se.statistics.nr_forced_migrations); } - return 1; } @@ -6314,63 +6486,67 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) } /* - * move_one_task tries to move exactly one task from busiest to this_rq, as + * detach_task() -- detach the task for the migration specified in env + */ +static void detach_task(struct task_struct *p, struct lb_env *env) +{ + lockdep_assert_held(&env->src_rq->lock); + + deactivate_task(env->src_rq, p, 0); + p->on_rq = TASK_ON_RQ_MIGRATING; + set_task_cpu(p, env->dst_cpu); +} + +/* + * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as * part of active balancing operations within "domain". - * Returns 1 if successful and 0 otherwise. * - * Called with both runqueues locked. + * Returns a task if successful and NULL otherwise. */ -static int move_one_task(struct lb_env *env) +static struct task_struct *detach_one_task(struct lb_env *env) { struct task_struct *p, *n; + lockdep_assert_held(&env->src_rq->lock); + list_for_each_entry_safe(p, n, &env->src_rq->cfs_tasks, se.group_node) { if (!can_migrate_task(p, env)) continue; - move_task(p, env); + detach_task(p, env); + /* - * Right now, this is only the second place move_task() - * is called, so we can safely collect move_task() - * stats here rather than inside move_task(). + * Right now, this is only the second place where + * lb_gained[env->idle] is updated (other is detach_tasks) + * so we can safely collect stats here rather than + * inside detach_tasks(). */ schedstat_inc(env->sd, lb_gained[env->idle]); - per_cpu(dbs_boost_load_moved, env->dst_cpu) += pct_task_load(p); - - return 1; + return p; } - return 0; + return NULL; } -static unsigned long task_h_load(struct task_struct *p); - static const unsigned int sched_nr_migrate_break = 32; /* - * move_tasks tries to move up to imbalance weighted load from busiest to - * this_rq, as part of a balancing operation within domain "sd". - * Returns the number of pulled tasks if successful and 0 otherwise. + * detach_tasks() -- tries to detach up to imbalance weighted load from + * busiest_rq, as part of a balancing operation within domain "sd". * - * Called with both runqueues locked. + * Returns number of detached tasks if successful and 0 otherwise. */ -static int move_tasks(struct lb_env *env) +static int detach_tasks(struct lb_env *env) { struct list_head *tasks = &env->src_rq->cfs_tasks; struct task_struct *p; unsigned long load; - int pulled = 0; - int orig_loop = env->loop; + int detached = 0; + + lockdep_assert_held(&env->src_rq->lock); if (env->imbalance <= 0) return 0; - if (capacity(env->dst_rq) > capacity(env->src_rq)) - env->flags |= LBF_IGNORE_SMALL_TASKS; - else if (capacity(env->dst_rq) < capacity(env->src_rq) && - !sched_boost()) - env->flags |= LBF_IGNORE_BIG_TASKS; - -redo: while (!list_empty(tasks)) { p = list_first_entry(tasks, struct task_struct, se.group_node); @@ -6397,15 +6573,16 @@ static int move_tasks(struct lb_env *env) if ((load / 2) > env->imbalance) goto next; - move_task(p, env); - pulled++; + detach_task(p, env); + list_add(&p->se.group_node, &env->tasks); + + detached++; env->imbalance -= load; - per_cpu(dbs_boost_load_moved, env->dst_cpu) += pct_task_load(p); #ifdef CONFIG_PREEMPT /* * NEWIDLE balancing is a source of latency, so preemptible - * kernels will stop after the first task is pulled to minimize + * kernels will stop after the first task is detached to minimize * the critical section. */ if (env->idle == CPU_NEWLY_IDLE) @@ -6424,22 +6601,59 @@ static int move_tasks(struct lb_env *env) list_move_tail(&p->se.group_node, tasks); } - if (env->flags & (LBF_IGNORE_SMALL_TASKS | LBF_IGNORE_BIG_TASKS) - && !pulled) { - tasks = &env->src_rq->cfs_tasks; - env->flags &= ~(LBF_IGNORE_SMALL_TASKS | LBF_IGNORE_BIG_TASKS); - env->loop = orig_loop; - goto redo; - } - /* - * Right now, this is one of only two places move_task() is called, - * so we can safely collect move_task() stats here rather than - * inside move_task(). + * Right now, this is one of only two places we collect this stat + * so we can safely collect detach_one_task() stats here rather + * than inside detach_one_task(). */ - schedstat_add(env->sd, lb_gained[env->idle], pulled); + schedstat_add(env->sd, lb_gained[env->idle], detached); + + return detached; +} + +/* + * attach_task() -- attach the task detached by detach_task() to its new rq. + */ +static void attach_task(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_held(&rq->lock); + + BUG_ON(task_rq(p) != rq); + p->on_rq = TASK_ON_RQ_QUEUED; + activate_task(rq, p, 0); + check_preempt_curr(rq, p, 0); +} + +/* + * attach_one_task() -- attaches the task returned from detach_one_task() to + * its new rq. + */ +static void attach_one_task(struct rq *rq, struct task_struct *p) +{ + raw_spin_lock(&rq->lock); + attach_task(rq, p); + raw_spin_unlock(&rq->lock); +} + +/* + * attach_tasks() -- attaches all tasks detached by detach_tasks() to their + * new rq. + */ +static void attach_tasks(struct lb_env *env) +{ + struct list_head *tasks = &env->tasks; + struct task_struct *p; + + raw_spin_lock(&env->dst_rq->lock); + + while (!list_empty(tasks)) { + p = list_first_entry(tasks, struct task_struct, se.group_node); + list_del_init(&p->se.group_node); + + attach_task(env->dst_rq, p); + } - return pulled; + raw_spin_unlock(&env->dst_rq->lock); } #ifdef CONFIG_FAIR_GROUP_SCHED @@ -6501,104 +6715,63 @@ static void update_blocked_averages(int cpu) } /* - * Compute the cpu's hierarchical load factor for each task group. + * Compute the hierarchical load factor for cfs_rq and all its ascendants. * This needs to be done in a top-down fashion because the load of a child * group is a fraction of its parents load. */ -static int tg_load_down(struct task_group *tg, void *data) -{ - unsigned long load; - long cpu = (long)data; - - if (!tg->parent) { - load = cpu_rq(cpu)->load.weight; - } else { - load = tg->parent->cfs_rq[cpu]->h_load; - load *= tg->se[cpu]->load.weight; - load /= tg->parent->cfs_rq[cpu]->load.weight + 1; - } - - tg->cfs_rq[cpu]->h_load = load; - - return 0; -} - -static void update_h_load(long cpu) +static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) { - struct rq *rq = cpu_rq(cpu); + struct rq *rq = rq_of(cfs_rq); + struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; unsigned long now = jiffies; + unsigned long load; - if (rq->h_load_throttle == now) + if (cfs_rq->last_h_load_update == now) return; - rq->h_load_throttle = now; + cfs_rq->h_load_next = NULL; + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + cfs_rq->h_load_next = se; + if (cfs_rq->last_h_load_update == now) + break; + } + + if (!se) { + cfs_rq->h_load = cfs_rq->runnable_load_avg; + cfs_rq->last_h_load_update = now; + } - rcu_read_lock(); - walk_tg_tree(tg_load_down, tg_nop, (void *)cpu); - rcu_read_unlock(); + while ((se = cfs_rq->h_load_next) != NULL) { + load = cfs_rq->h_load; + load = div64_ul(load * se->avg.load_avg_contrib, + cfs_rq->runnable_load_avg + 1); + cfs_rq = group_cfs_rq(se); + cfs_rq->h_load = load; + cfs_rq->last_h_load_update = now; + } } static unsigned long task_h_load(struct task_struct *p) { struct cfs_rq *cfs_rq = task_cfs_rq(p); - unsigned long load; - load = p->se.load.weight; - load = div_u64(load * cfs_rq->h_load, cfs_rq->load.weight + 1); - - return load; + update_cfs_rq_h_load(cfs_rq); + return div64_ul(p->se.avg.load_avg_contrib * cfs_rq->h_load, + cfs_rq->runnable_load_avg + 1); } #else static inline void update_blocked_averages(int cpu) { } -static inline void update_h_load(long cpu) -{ -} - static unsigned long task_h_load(struct task_struct *p) { - return p->se.load.weight; + return p->se.avg.load_avg_contrib; } #endif /********** Helpers for find_busiest_group ************************/ -/* - * sd_lb_stats - Structure to store the statistics of a sched_domain - * during load balancing. - */ -struct sd_lb_stats { - struct sched_group *busiest; /* Busiest group in this sd */ - struct sched_group *this; /* Local group in this sd */ - unsigned long total_load; /* Total load of all groups in sd */ - unsigned long total_pwr; /* Total power of all groups in sd */ - unsigned long avg_load; /* Average load across all groups in sd */ - - /** Statistics of this group */ - unsigned long this_load; - unsigned long this_load_per_task; - unsigned long this_nr_running; - unsigned long this_has_capacity; - unsigned long this_group_capacity; - unsigned int this_idle_cpus; - - /* Statistics of the busiest group */ - unsigned int busiest_idle_cpus; - unsigned long max_load; - unsigned long busiest_load_per_task; - unsigned long busiest_nr_running; -#ifdef CONFIG_SCHED_HMP - unsigned long busiest_nr_small_tasks; - unsigned long busiest_nr_big_tasks; - u64 busiest_scaled_load; -#endif - unsigned long busiest_group_capacity; - unsigned long busiest_has_capacity; - unsigned int busiest_group_weight; - - int group_imb; /* Is there imbalance in this sd */ -}; /* * sg_lb_stats - stats of a sched_group required for load_balancing @@ -6606,57 +6779,64 @@ struct sd_lb_stats { struct sg_lb_stats { unsigned long avg_load; /*Avg load across the CPUs of the group */ unsigned long group_load; /* Total load over the CPUs of the group */ - unsigned long sum_nr_running; /* Nr tasks running in the group */ -#ifdef CONFIG_SCHED_HMP - unsigned long sum_nr_big_tasks, sum_nr_small_tasks; - u64 group_cpu_load; /* Scaled load of all CPUs of the group */ -#endif unsigned long sum_weighted_load; /* Weighted load of group's tasks */ + unsigned long load_per_task; unsigned long group_capacity; - unsigned long idle_cpus; - unsigned long group_weight; - int group_imb; /* Is there an imbalance in the group ? */ - int group_has_capacity; /* Is there extra capacity in the group? */ + unsigned long group_usage; /* Total usage of the group */ + unsigned int sum_nr_running; /* Nr tasks running in the group */ + unsigned int idle_cpus; + unsigned int group_weight; + enum group_type group_type; + int group_no_capacity; + int group_misfit_task; /* A cpu has a task too big for its capacity */ +#ifdef CONFIG_NUMA_BALANCING + unsigned int nr_numa_running; + unsigned int nr_preferred_running; +#endif }; -#ifdef CONFIG_SCHED_HMP - -static int -bail_inter_cluster_balance(struct lb_env *env, struct sd_lb_stats *sds) -{ - int nr_cpus; - - if (group_rq_capacity(sds->this) <= group_rq_capacity(sds->busiest)) - return 0; - - if (sds->busiest_nr_big_tasks) - return 0; - - nr_cpus = cpumask_weight(sched_group_cpus(sds->busiest)); - - if ((sds->busiest_scaled_load < nr_cpus * sched_spill_load) && - (sds->busiest_nr_running < - nr_cpus * sysctl_sched_spill_nr_run)) { - return 1; - } - - return 0; -} - -#else /* CONFIG_SCHED_HMP */ +/* + * sd_lb_stats - Structure to store the statistics of a sched_domain + * during load balancing. + */ +struct sd_lb_stats { + struct sched_group *busiest; /* Busiest group in this sd */ + struct sched_group *local; /* Local group in this sd */ + unsigned long total_load; /* Total load of all groups in sd */ + unsigned long total_capacity; /* Total capacity of all groups in sd */ + unsigned long avg_load; /* Average load across all groups in sd */ + + struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */ + struct sg_lb_stats local_stat; /* Statistics of the local group */ +}; -static inline int -bail_inter_cluster_balance(struct lb_env *env, struct sd_lb_stats *sds) +static inline void init_sd_lb_stats(struct sd_lb_stats *sds) { - return 0; + /* + * Skimp on the clearing to avoid duplicate work. We can avoid clearing + * local_stat because update_sg_lb_stats() does a full clear/assignment. + * We must however clear busiest_stat::avg_load because + * update_sd_pick_busiest() reads this before assignment. + */ + *sds = (struct sd_lb_stats){ + .busiest = NULL, + .local = NULL, + .total_load = 0UL, + .total_capacity = 0UL, + .busiest_stat = { + .avg_load = 0UL, + .sum_nr_running = 0, + .group_type = group_other, + }, + }; } -#endif /* CONFIG_SCHED_HMP */ - /** * get_sd_load_idx - Obtain the load index for a given sched domain. * @sd: The sched_domain whose load_idx is to be obtained. - * @idle: The Idle status of the CPU for whose sd load_icx is obtained. + * @idle: The idle status of the CPU for whose sd load_idx is obtained. + * + * Return: The load index. */ static inline int get_sd_load_idx(struct sched_domain *sd, enum cpu_idle_type idle) @@ -6679,35 +6859,21 @@ static inline int get_sd_load_idx(struct sched_domain *sd, return load_idx; } -static unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu) -{ - return SCHED_POWER_SCALE; -} - -unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu) -{ - return default_scale_freq_power(sd, cpu); -} - -static unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu) +static unsigned long default_scale_capacity(struct sched_domain *sd, int cpu) { - unsigned long weight = sd->span_weight; - unsigned long smt_gain = sd->smt_gain; - - smt_gain /= weight; - - return smt_gain; + return SCHED_CAPACITY_SCALE; } -unsigned long __weak arch_scale_smt_power(struct sched_domain *sd, int cpu) +unsigned long __weak arch_scale_freq_capacity(struct sched_domain *sd, int cpu) { - return default_scale_smt_power(sd, cpu); + return default_scale_capacity(sd, cpu); } -static unsigned long scale_rt_power(int cpu) +static unsigned long scale_rt_capacity(int cpu) { struct rq *rq = cpu_rq(cpu); - u64 total, available, age_stamp, avg; + u64 total, used, age_stamp, avg; + s64 delta; /* * Since we're reading these variables without serialization make sure @@ -6715,75 +6881,56 @@ static unsigned long scale_rt_power(int cpu) */ age_stamp = ACCESS_ONCE(rq->age_stamp); avg = ACCESS_ONCE(rq->rt_avg); + delta = __rq_clock_broken(rq) - age_stamp; - total = sched_avg_period() + (rq->clock - age_stamp); + if (unlikely(delta < 0)) + delta = 0; - if (unlikely(total < avg)) { - /* Ensures that power won't end up being negative */ - available = 0; - } else { - available = total - avg; - } + total = sched_avg_period() + delta; - if (unlikely((s64)total < SCHED_POWER_SCALE)) - total = SCHED_POWER_SCALE; + used = div_u64(avg, total); - total >>= SCHED_POWER_SHIFT; + if (likely(used < SCHED_CAPACITY_SCALE)) + return SCHED_CAPACITY_SCALE - used; - return div_u64(available, total); + return 1; } -static void update_cpu_power(struct sched_domain *sd, int cpu) +static void update_cpu_capacity(struct sched_domain *sd, int cpu) { - unsigned long weight = sd->span_weight; - unsigned long power = SCHED_POWER_SCALE; + unsigned long capacity = arch_scale_cpu_capacity(sd, cpu); struct sched_group *sdg = sd->groups; + cpu_rq(cpu)->cpu_capacity_orig = capacity; - if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) { - if (sched_feat(ARCH_POWER)) - power *= arch_scale_smt_power(sd, cpu); - else - power *= default_scale_smt_power(sd, cpu); - - power >>= SCHED_POWER_SHIFT; - } - - sdg->sgp->power_orig = power; - - if (sched_feat(ARCH_POWER)) - power *= arch_scale_freq_power(sd, cpu); - else - power *= default_scale_freq_power(sd, cpu); + capacity *= scale_rt_capacity(cpu); + capacity >>= SCHED_CAPACITY_SHIFT; - power >>= SCHED_POWER_SHIFT; + if (!capacity) + capacity = 1; - power *= scale_rt_power(cpu); - power >>= SCHED_POWER_SHIFT; - - if (!power) - power = 1; - - cpu_rq(cpu)->cpu_power = power; - sdg->sgp->power = power; + cpu_rq(cpu)->cpu_capacity = capacity; + sdg->sgc->capacity = capacity; + sdg->sgc->max_capacity = capacity; } -void update_group_power(struct sched_domain *sd, int cpu) +void update_group_capacity(struct sched_domain *sd, int cpu) { struct sched_domain *child = sd->child; struct sched_group *group, *sdg = sd->groups; - unsigned long power; + unsigned long capacity, max_capacity; unsigned long interval; interval = msecs_to_jiffies(sd->balance_interval); interval = clamp(interval, 1UL, max_load_balance_interval); - sdg->sgp->next_update = jiffies + interval; + sdg->sgc->next_update = jiffies + interval; if (!child) { - update_cpu_power(sd, cpu); + update_cpu_capacity(sd, cpu); return; } - power = 0; + capacity = 0; + max_capacity = 0; if (child->flags & SD_OVERLAP) { /* @@ -6791,47 +6938,166 @@ void update_group_power(struct sched_domain *sd, int cpu) * span the current group. */ - for_each_cpu(cpu, sched_group_cpus(sdg)) - power += power_of(cpu); + for_each_cpu(cpu, sched_group_cpus(sdg)) { + struct sched_group_capacity *sgc; + struct rq *rq = cpu_rq(cpu); + + /* + * build_sched_domains() -> init_sched_groups_capacity() + * gets here before we've attached the domains to the + * runqueues. + * + * Use capacity_of(), which is set irrespective of domains + * in update_cpu_capacity(). + * + * This avoids capacity from being 0 and + * causing divide-by-zero issues on boot. + */ + if (unlikely(!rq->sd)) { + capacity += capacity_of(cpu); + } else { + sgc = rq->sd->groups->sgc; + capacity += sgc->capacity; + } + + max_capacity = max(capacity, max_capacity); + } } else { /* * !SD_OVERLAP domains can assume that child groups * span the current group. */ - group = child->groups; - do { - power += group->sgp->power; - group = group->next; - } while (group != child->groups); - } + group = child->groups; + do { + struct sched_group_capacity *sgc = group->sgc; + + capacity += sgc->capacity; + max_capacity = max(sgc->max_capacity, max_capacity); + group = group->next; + } while (group != child->groups); + } + + sdg->sgc->capacity = capacity; + sdg->sgc->max_capacity = max_capacity; +} + +/* + * Check whether the capacity of the rq has been noticeably reduced by side + * activity. The imbalance_pct is used for the threshold. + * Return true is the capacity is reduced + */ +static inline int +check_cpu_capacity(struct rq *rq, struct sched_domain *sd) +{ + return ((rq->cpu_capacity * sd->imbalance_pct) < + (rq->cpu_capacity_orig * 100)); +} + +/* + * Group imbalance indicates (and tries to solve) the problem where balancing + * groups is inadequate due to tsk_cpus_allowed() constraints. + * + * Imagine a situation of two groups of 4 cpus each and 4 tasks each with a + * cpumask covering 1 cpu of the first group and 3 cpus of the second group. + * Something like: + * + * { 0 1 2 3 } { 4 5 6 7 } + * * * * * + * + * If we were to balance group-wise we'd place two tasks in the first group and + * two tasks in the second group. Clearly this is undesired as it will overload + * cpu 3 and leave one of the cpus in the second group unused. + * + * The current solution to this issue is detecting the skew in the first group + * by noticing the lower domain failed to reach balance and had difficulty + * moving tasks due to affinity constraints. + * + * When this is so detected; this group becomes a candidate for busiest; see + * update_sd_pick_busiest(). And calculate_imbalance() and + * find_busiest_group() avoid some of the usual balance conditions to allow it + * to create an effective group imbalance. + * + * This is a somewhat tricky proposition since the next run might not find the + * group imbalance and decide the groups need to be balanced again. A most + * subtle and fragile situation. + */ + +static inline int sg_imbalanced(struct sched_group *group) +{ + return group->sgc->imbalance; +} + +/* + * group_has_capacity returns true if the group has spare capacity that could + * be used by some tasks. + * We consider that a group has spare capacity if the * number of task is + * smaller than the number of CPUs or if the usage is lower than the available + * capacity for CFS tasks. + * For the latter, we use a threshold to stabilize the state, to take into + * account the variance of the tasks' load and to return true if the available + * capacity in meaningful for the load balancer. + * As an example, an available capacity of 1% can appear but it doesn't make + * any benefit for the load balance. + */ +static inline bool +group_has_capacity(struct lb_env *env, struct sg_lb_stats *sgs) +{ + if (sgs->sum_nr_running < sgs->group_weight) + return true; + + if ((sgs->group_capacity * 100) > + (sgs->group_usage * env->sd->imbalance_pct)) + return true; + + return false; +} + +/* + * group_is_overloaded returns true if the group has more tasks than it can + * handle. + * group_is_overloaded is not equals to !group_has_capacity because a group + * with the exact right number of tasks, has no more spare capacity but is not + * overloaded so both group_has_capacity and group_is_overloaded return + * false. + */ +static inline bool +group_is_overloaded(struct lb_env *env, struct sg_lb_stats *sgs) +{ + if (sgs->sum_nr_running <= sgs->group_weight) + return false; + + if ((sgs->group_capacity * 100) < + (sgs->group_usage * env->sd->imbalance_pct)) + return true; - sdg->sgp->power_orig = sdg->sgp->power = power; + return false; } /* - * Try and fix up capacity for tiny siblings, this is needed when - * things like SD_ASYM_PACKING need f_b_g to select another sibling - * which on its own isn't powerful enough. - * - * See update_sd_pick_busiest() and check_asym_packing(). + * group_smaller_cpu_capacity: Returns true if sched_group sg has smaller + * per-cpu capacity than sched_group ref. */ -static inline int -fix_small_capacity(struct sched_domain *sd, struct sched_group *group) +static inline bool +group_smaller_cpu_capacity(struct sched_group *sg, struct sched_group *ref) { - /* - * Only siblings can have significantly less than SCHED_POWER_SCALE - */ - if (!(sd->flags & SD_SHARE_CPUPOWER)) - return 0; + return sg->sgc->max_capacity + sysctl_sched_capacity_margin - SCHED_LOAD_SCALE < + ref->sgc->max_capacity; +} - /* - * If ~90% of the cpu_power is still there, we're good. - */ - if (group->sgp->power * 32 > group->sgp->power_orig * 29) - return 1; +static enum group_type group_classify(struct lb_env *env, + struct sched_group *group, + struct sg_lb_stats *sgs) +{ + if (sgs->group_no_capacity) + return group_overloaded; - return 0; + if (sg_imbalanced(group)) + return group_imbalanced; + + if (sgs->group_misfit_task) + return group_misfit_task; + return group_other; } /** @@ -6840,148 +7106,63 @@ fix_small_capacity(struct sched_domain *sd, struct sched_group *group) * @group: sched_group whose statistics are to be updated. * @load_idx: Load index of sched_domain of this_cpu for load calc. * @local_group: Does group contain this_cpu. - * @balance: Should we balance. * @sgs: variable to hold the statistics for this group. + * @overload: Indicate more than one runnable task for any CPU. + * @overutilized: Indicate overutilization for any CPU. */ static inline void update_sg_lb_stats(struct lb_env *env, struct sched_group *group, int load_idx, - int local_group, int *balance, struct sg_lb_stats *sgs) + int local_group, struct sg_lb_stats *sgs, + bool *overload, bool *overutilized) { - unsigned long nr_running, max_nr_running, min_nr_running; - unsigned long load, max_cpu_load, min_cpu_load; - unsigned int balance_cpu = -1, first_idle_cpu = 0; - unsigned long avg_load_per_task = 0; + unsigned long load; int i; - if (local_group) - balance_cpu = group_balance_cpu(group); - - /* Tally up the load of all CPUs in the group */ - max_cpu_load = 0; - min_cpu_load = ~0UL; - max_nr_running = 0; - min_nr_running = ~0UL; + memset(sgs, 0, sizeof(*sgs)); for_each_cpu_and(i, sched_group_cpus(group), env->cpus) { struct rq *rq = cpu_rq(i); - trace_sched_cpu_load(cpu_rq(i), idle_cpu(i), - mostly_idle_cpu(i), - sched_irqload(i), - power_cost_at_freq(i, 0), - cpu_temp(i)); - nr_running = rq->nr_running; - /* Bias balancing toward cpus of our domain */ - if (local_group) { - if (idle_cpu(i) && !first_idle_cpu && - cpumask_test_cpu(i, sched_group_mask(group))) { - first_idle_cpu = 1; - balance_cpu = i; - } - + if (local_group) load = target_load(i, load_idx); - } else { + else load = source_load(i, load_idx); - if (load > max_cpu_load) - max_cpu_load = load; - if (min_cpu_load > load) - min_cpu_load = load; - - if (nr_running > max_nr_running) - max_nr_running = nr_running; - if (min_nr_running > nr_running) - min_nr_running = nr_running; - } sgs->group_load += load; - sgs->sum_nr_running += nr_running; -#ifdef CONFIG_SCHED_HMP - sgs->sum_nr_big_tasks += rq->hmp_stats.nr_big_tasks; - sgs->sum_nr_small_tasks += rq->hmp_stats.nr_small_tasks; - sgs->group_cpu_load += cpu_load(i); + sgs->group_usage += get_cpu_usage(i); + sgs->sum_nr_running += rq->cfs.h_nr_running; + + if (rq->nr_running > 1) + *overload = true; + +#ifdef CONFIG_NUMA_BALANCING + sgs->nr_numa_running += rq->nr_numa_running; + sgs->nr_preferred_running += rq->nr_preferred_running; #endif sgs->sum_weighted_load += weighted_cpuload(i); if (idle_cpu(i)) sgs->idle_cpus++; - } - /* - * First idle cpu or the first cpu(busiest) in this sched group - * is eligible for doing load balancing at this and above - * domains. In the newly idle case, we will allow all the cpu's - * to do the newly idle load balance. - */ - if (local_group) { - if (env->idle != CPU_NEWLY_IDLE) { - if (balance_cpu != env->dst_cpu) { - *balance = 0; - return; - } - update_group_power(env->sd, env->dst_cpu); - } else if (time_after_eq(jiffies, group->sgp->next_update)) - update_group_power(env->sd, env->dst_cpu); + if (cpu_overutilized(i)) { + *overutilized = true; + if (!sgs->group_misfit_task && rq->misfit_task) + sgs->group_misfit_task = capacity_of(i); + } } - /* Adjust by relative CPU power of the group */ - sgs->avg_load = (sgs->group_load*SCHED_POWER_SCALE) / group->sgp->power; + /* Adjust by relative CPU capacity of the group */ + sgs->group_capacity = group->sgc->capacity; + sgs->avg_load = (sgs->group_load*SCHED_CAPACITY_SCALE) / sgs->group_capacity; - /* - * Consider the group unbalanced when the imbalance is larger - * than the average weight of a task. - * - * APZ: with cgroup the avg task weight can vary wildly and - * might not be a suitable number - should we keep a - * normalized nr_running number somewhere that negates - * the hierarchy? - */ if (sgs->sum_nr_running) - avg_load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running; - - if ((max_cpu_load - min_cpu_load) >= avg_load_per_task && - (max_nr_running - min_nr_running) > 1) - sgs->group_imb = 1; + sgs->load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running; - sgs->group_capacity = DIV_ROUND_CLOSEST(group->sgp->power, - SCHED_POWER_SCALE); - if (!sgs->group_capacity) - sgs->group_capacity = fix_small_capacity(env->sd, group); sgs->group_weight = group->group_weight; - if (sgs->group_capacity > sgs->sum_nr_running) - sgs->group_has_capacity = 1; -} - -#ifdef CONFIG_SCHED_HMP -static bool update_sd_pick_busiest_active_balance(struct lb_env *env, - struct sd_lb_stats *sds, - struct sched_group *sg, - struct sg_lb_stats *sgs) -{ - if (env->idle != CPU_NOT_IDLE && - capacity(env->dst_rq) > group_rq_capacity(sg)) { - if (sched_boost() && !sds->busiest && sgs->sum_nr_running) { - env->flags |= LBF_SCHED_BOOST_ACTIVE_BALANCE; - return true; - } - - if (sgs->sum_nr_big_tasks > sds->busiest_nr_big_tasks) { - env->flags |= LBF_BIG_TASK_ACTIVE_BALANCE; - return true; - } - } - - return false; -} -#else -static bool update_sd_pick_busiest_active_balance(struct lb_env *env, - struct sd_lb_stats *sds, - struct sched_group *sg, - struct sg_lb_stats *sgs) -{ - return false; + sgs->group_no_capacity = group_is_overloaded(env, sgs); + sgs->group_type = group_classify(env, group, sgs); } -#endif /** * update_sd_pick_busiest - return 1 on busiest group @@ -6992,67 +7173,52 @@ static bool update_sd_pick_busiest_active_balance(struct lb_env *env, * * Determine if @sg is a busier group than the previously selected * busiest group. + * + * Return: %true if @sg is a busier group than the previously selected + * busiest group. %false otherwise. */ static bool update_sd_pick_busiest(struct lb_env *env, struct sd_lb_stats *sds, struct sched_group *sg, struct sg_lb_stats *sgs) { - int cpu, cpu_busiest; - unsigned int pc, pc_busiest; + struct sg_lb_stats *busiest = &sds->busiest_stat; - if (update_sd_pick_busiest_active_balance(env, sds, sg, sgs)) + if (sgs->group_type > busiest->group_type) return true; - if (sgs->avg_load < sds->max_load) + if (sgs->group_type < busiest->group_type) return false; - if (sgs->sum_nr_running > sgs->group_capacity) { - env->flags &= ~LBF_EA_ACTIVE_BALANCE; - return true; - } + /* + * Candidate sg doesn't face any serious load-balance problems + * so don't pick it if the local sg is already filled up. + */ + if (sgs->group_type == group_other && + !group_has_capacity(env, &sds->local_stat)) + return false; - if (sgs->group_imb) { - env->flags &= ~LBF_EA_ACTIVE_BALANCE; - return true; - } + if (sgs->avg_load <= busiest->avg_load) + return false; /* - * Mark a less power-efficient CPU as busy only we are close to - * throttling. We want to prioritize spreading work over power - * optimization. - */ - cpu = group_first_cpu(sg); - if (sysctl_sched_enable_power_aware && - (!sds->busiest || (env->flags & LBF_EA_ACTIVE_BALANCE)) && - (capacity(env->dst_rq) == group_rq_capacity(sg)) && - sgs->sum_nr_running && (env->idle != CPU_NOT_IDLE) && - !is_task_migration_throttled(cpu_rq(cpu)->curr) && - is_cpu_throttling_imminent(cpu)) { - pc = power_cost_at_freq(cpu, 0); - if (sds->busiest) { - cpu_busiest = group_first_cpu(sds->busiest); - pc_busiest = power_cost_at_freq(cpu_busiest, 0); - if (pc_busiest < pc) - return true; - } else { - if (power_cost_at_freq(env->dst_cpu, 0) < pc) { - env->flags |= LBF_EA_ACTIVE_BALANCE; - return true; - } - } - } - - if (sgs->avg_load == sds->max_load) + * Candiate sg has no more than one task per cpu and has higher + * per-cpu capacity. No reason to pull tasks to less capable cpus. + */ + if (sgs->sum_nr_running <= sgs->group_weight && + group_smaller_cpu_capacity(sds->local, sg)) return false; + /* This is the busiest node in its class. */ + if (!(env->sd->flags & SD_ASYM_PACKING)) + return true; + /* * ASYM_PACKING needs to move all the work to the lowest * numbered CPUs in the group, therefore mark all groups * higher than ourself as busy. */ - if ((env->sd->flags & SD_ASYM_PACKING) && sgs->sum_nr_running && - env->dst_cpu < group_first_cpu(sg)) { + if (sgs->sum_nr_running && env->dst_cpu < group_first_cpu(sg)) { if (!sds->busiest) return true; @@ -7063,19 +7229,51 @@ static bool update_sd_pick_busiest(struct lb_env *env, return false; } +#ifdef CONFIG_NUMA_BALANCING +static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) +{ + if (sgs->sum_nr_running > sgs->nr_numa_running) + return regular; + if (sgs->sum_nr_running > sgs->nr_preferred_running) + return remote; + return all; +} + +static inline enum fbq_type fbq_classify_rq(struct rq *rq) +{ + if (rq->nr_running > rq->nr_numa_running) + return regular; + if (rq->nr_running > rq->nr_preferred_running) + return remote; + return all; +} +#else +static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) +{ + return all; +} + +static inline enum fbq_type fbq_classify_rq(struct rq *rq) +{ + return regular; +} +#endif /* CONFIG_NUMA_BALANCING */ + +#define lb_sd_parent(sd) \ + (sd->parent && sd->parent->groups != sd->parent->groups->next) + /** * update_sd_lb_stats - Update sched_domain's statistics for load balancing. * @env: The load balancing environment. - * @balance: Should we balance. * @sds: variable to hold the statistics for this sched_domain. */ -static inline void update_sd_lb_stats(struct lb_env *env, - int *balance, struct sd_lb_stats *sds) +static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds) { struct sched_domain *child = env->sd->child; struct sched_group *sg = env->sd->groups; - struct sg_lb_stats sgs; + struct sg_lb_stats tmp_sgs; int load_idx, prefer_sibling = 0; + bool overload = false, overutilized = false; if (child && child->flags & SD_PREFER_SIBLING) prefer_sibling = 1; @@ -7083,60 +7281,81 @@ static inline void update_sd_lb_stats(struct lb_env *env, load_idx = get_sd_load_idx(env->sd, env->idle); do { + struct sg_lb_stats *sgs = &tmp_sgs; int local_group; local_group = cpumask_test_cpu(env->dst_cpu, sched_group_cpus(sg)); - memset(&sgs, 0, sizeof(sgs)); - update_sg_lb_stats(env, sg, load_idx, local_group, balance, &sgs); + if (local_group) { + sds->local = sg; + sgs = &sds->local_stat; - if (local_group && !(*balance)) - return; + if (env->idle != CPU_NEWLY_IDLE || + time_after_eq(jiffies, sg->sgc->next_update)) + update_group_capacity(env->sd, env->dst_cpu); + } - sds->total_load += sgs.group_load; - sds->total_pwr += sg->sgp->power; + update_sg_lb_stats(env, sg, load_idx, local_group, sgs, + &overload, &overutilized); + + if (local_group) + goto next_group; /* * In case the child domain prefers tasks go to siblings - * first, lower the sg capacity to one so that we'll try + * first, lower the sg capacity so that we'll try * and move all the excess tasks away. We lower the capacity * of a group only if the local group has the capacity to fit - * these excess tasks, i.e. nr_running < group_capacity. The - * extra check prevents the case where you always pull from the - * heaviest group when it is already under-utilized (possible - * with a large weight task outweighs the tasks on the system). + * these excess tasks. The extra check prevents the case where + * you always pull from the heaviest group when it is already + * under-utilized (possible with a large weight task outweighs + * the tasks on the system). + */ + if (prefer_sibling && sds->local && + group_has_capacity(env, &sds->local_stat) && + (sgs->sum_nr_running > 1)) { + sgs->group_no_capacity = 1; + sgs->group_type = group_overloaded; + } + + /* + * Ignore task groups with misfit tasks if local group has no + * capacity or if per-cpu capacity isn't higher. */ - if (prefer_sibling && !local_group && sds->this_has_capacity) - sgs.group_capacity = min(sgs.group_capacity, 1UL); + if (sgs->group_type == group_misfit_task && + (!group_has_capacity(env, &sds->local_stat) || + !group_smaller_cpu_capacity(sg, sds->local))) + sgs->group_type = group_other; - if (local_group) { - sds->this_load = sgs.avg_load; - sds->this = sg; - sds->this_nr_running = sgs.sum_nr_running; - sds->this_load_per_task = sgs.sum_weighted_load; - sds->this_has_capacity = sgs.group_has_capacity; - sds->this_idle_cpus = sgs.idle_cpus; - sds->this_group_capacity = sgs.group_capacity; - } else if (update_sd_pick_busiest(env, sds, sg, &sgs)) { - sds->max_load = sgs.avg_load; + if (update_sd_pick_busiest(env, sds, sg, sgs)) { sds->busiest = sg; - env->busiest_nr_running = sds->busiest_nr_running - = sgs.sum_nr_running; - sds->busiest_idle_cpus = sgs.idle_cpus; - env->busiest_grp_capacity = sds->busiest_group_capacity - = sgs.group_capacity; - sds->busiest_load_per_task = sgs.sum_weighted_load; - sds->busiest_has_capacity = sgs.group_has_capacity; - sds->busiest_group_weight = sgs.group_weight; - sds->group_imb = sgs.group_imb; -#ifdef CONFIG_SCHED_HMP - sds->busiest_nr_small_tasks = sgs.sum_nr_small_tasks; - sds->busiest_nr_big_tasks = sgs.sum_nr_big_tasks; - sds->busiest_scaled_load = sgs.group_cpu_load; -#endif + sds->busiest_stat = *sgs; } +next_group: + /* Now, start updating sd_lb_stats */ + sds->total_load += sgs->group_load; + sds->total_capacity += sgs->group_capacity; + sg = sg->next; } while (sg != env->sd->groups); + + if (env->sd->flags & SD_NUMA) + env->fbq_type = fbq_classify_group(&sds->busiest_stat); + + env->src_grp_nr_running = sds->busiest_stat.sum_nr_running; + + if (!lb_sd_parent(env->sd)) { + /* update overload indicator if we are at root domain */ + if (env->dst_rq->rd->overload != overload) + env->dst_rq->rd->overload = overload; + + /* Update over-utilization (tipping point, U >= 0) indicator */ + if (env->dst_rq->rd->overutilized != overutilized) + env->dst_rq->rd->overutilized = overutilized; + } else { + if (!env->dst_rq->rd->overutilized && overutilized) + env->dst_rq->rd->overutilized = true; + } } /** @@ -7156,7 +7375,7 @@ static inline void update_sd_lb_stats(struct lb_env *env, * assuming lower CPU number will be equivalent to lower a SMT thread * number. * - * Returns 1 when packing is required and a task should be moved to + * Return: 1 when packing is required and a task should be moved to * this CPU. The amount of the imbalance is returned in *imbalance. * * @env: The load balancing environment. @@ -7177,7 +7396,8 @@ static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds) return 0; env->imbalance = DIV_ROUND_CLOSEST( - sds->max_load * sds->busiest->sgp->power, SCHED_POWER_SCALE); + sds->busiest_stat.avg_load * sds->busiest_stat.group_capacity, + SCHED_CAPACITY_SCALE); return 1; } @@ -7192,64 +7412,64 @@ static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds) static inline void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds) { - unsigned long tmp, pwr_now = 0, pwr_move = 0; + unsigned long tmp, capa_now = 0, capa_move = 0; unsigned int imbn = 2; unsigned long scaled_busy_load_per_task; + struct sg_lb_stats *local, *busiest; - if (sds->this_nr_running) { - sds->this_load_per_task /= sds->this_nr_running; - if (sds->busiest_load_per_task > - sds->this_load_per_task) - imbn = 1; - } else { - sds->this_load_per_task = - cpu_avg_load_per_task(env->dst_cpu); - } + local = &sds->local_stat; + busiest = &sds->busiest_stat; - scaled_busy_load_per_task = sds->busiest_load_per_task - * SCHED_POWER_SCALE; - scaled_busy_load_per_task /= sds->busiest->sgp->power; + if (!local->sum_nr_running) + local->load_per_task = cpu_avg_load_per_task(env->dst_cpu); + else if (busiest->load_per_task > local->load_per_task) + imbn = 1; - if (sds->max_load - sds->this_load + scaled_busy_load_per_task >= - (scaled_busy_load_per_task * imbn)) { - env->imbalance = sds->busiest_load_per_task; + scaled_busy_load_per_task = + (busiest->load_per_task * SCHED_CAPACITY_SCALE) / + busiest->group_capacity; + + if (busiest->avg_load + scaled_busy_load_per_task >= + local->avg_load + (scaled_busy_load_per_task * imbn)) { + env->imbalance = busiest->load_per_task; return; } /* * OK, we don't have enough imbalance to justify moving tasks, - * however we may be able to increase total CPU power used by + * however we may be able to increase total CPU capacity used by * moving them. */ - pwr_now += sds->busiest->sgp->power * - min(sds->busiest_load_per_task, sds->max_load); - pwr_now += sds->this->sgp->power * - min(sds->this_load_per_task, sds->this_load); - pwr_now /= SCHED_POWER_SCALE; + capa_now += busiest->group_capacity * + min(busiest->load_per_task, busiest->avg_load); + capa_now += local->group_capacity * + min(local->load_per_task, local->avg_load); + capa_now /= SCHED_CAPACITY_SCALE; /* Amount of load we'd subtract */ - tmp = (sds->busiest_load_per_task * SCHED_POWER_SCALE) / - sds->busiest->sgp->power; - if (sds->max_load > tmp) - pwr_move += sds->busiest->sgp->power * - min(sds->busiest_load_per_task, sds->max_load - tmp); + if (busiest->avg_load > scaled_busy_load_per_task) { + capa_move += busiest->group_capacity * + min(busiest->load_per_task, + busiest->avg_load - scaled_busy_load_per_task); + } /* Amount of load we'd add */ - if (sds->max_load * sds->busiest->sgp->power < - sds->busiest_load_per_task * SCHED_POWER_SCALE) - tmp = (sds->max_load * sds->busiest->sgp->power) / - sds->this->sgp->power; - else - tmp = (sds->busiest_load_per_task * SCHED_POWER_SCALE) / - sds->this->sgp->power; - pwr_move += sds->this->sgp->power * - min(sds->this_load_per_task, sds->this_load + tmp); - pwr_move /= SCHED_POWER_SCALE; + if (busiest->avg_load * busiest->group_capacity < + busiest->load_per_task * SCHED_CAPACITY_SCALE) { + tmp = (busiest->avg_load * busiest->group_capacity) / + local->group_capacity; + } else { + tmp = (busiest->load_per_task * SCHED_CAPACITY_SCALE) / + local->group_capacity; + } + capa_move += local->group_capacity * + min(local->load_per_task, local->avg_load + tmp); + capa_move /= SCHED_CAPACITY_SCALE; /* Move if we gain throughput */ - if (pwr_move > pwr_now) - env->imbalance = sds->busiest_load_per_task; + if (capa_move > capa_now) + env->imbalance = busiest->load_per_task; } /** @@ -7261,33 +7481,58 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds) static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) { unsigned long max_pull, load_above_capacity = ~0UL; + struct sg_lb_stats *local, *busiest; + + local = &sds->local_stat; + busiest = &sds->busiest_stat; - sds->busiest_load_per_task /= sds->busiest_nr_running; - if (sds->group_imb) { - sds->busiest_load_per_task = - min(sds->busiest_load_per_task, sds->avg_load); + if (busiest->group_type == group_imbalanced) { + /* + * In the group_imb case we cannot rely on group-wide averages + * to ensure cpu-load equilibrium, look at wider averages. XXX + */ + busiest->load_per_task = + min(busiest->load_per_task, sds->avg_load); } /* * In the presence of smp nice balancing, certain scenarios can have * max load less than avg load(as we skip the groups at or below - * its cpu_power, while calculating max_load..) + * its cpu_capacity, while calculating max_load..) */ - if (sds->max_load < sds->avg_load) { - env->imbalance = 0; - return fix_small_imbalance(env, sds); - } + if (busiest->avg_load <= sds->avg_load || + local->avg_load >= sds->avg_load) { + /* Misfitting tasks should be migrated in any case */ + if (busiest->group_type == group_misfit_task) { + env->imbalance = busiest->group_misfit_task; + return; + } - if (!sds->group_imb) { /* - * Don't want to pull so many tasks that a group would go idle. + * Busiest group is overloaded, local is not, use the spare + * cycles to maximize throughput */ - load_above_capacity = (sds->busiest_nr_running - - sds->busiest_group_capacity); + if (busiest->group_type == group_overloaded && + local->group_type <= group_misfit_task) { + env->imbalance = busiest->load_per_task; + return; + } - load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_POWER_SCALE); + env->imbalance = 0; + return fix_small_imbalance(env, sds); + } - load_above_capacity /= sds->busiest->sgp->power; + /* + * If there aren't any idle cpus, avoid creating some. + */ + if (busiest->group_type == group_overloaded && + local->group_type == group_overloaded) { + load_above_capacity = busiest->sum_nr_running * + SCHED_LOAD_SCALE; + if (load_above_capacity > busiest->group_capacity) + load_above_capacity -= busiest->group_capacity; + else + load_above_capacity = ~0UL; } /* @@ -7297,15 +7542,19 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s * we also don't want to reduce the group load below the group capacity * (so that we can implement power-savings policies etc). Thus we look * for the minimum possible imbalance. - * Be careful of negative numbers as they'll appear as very large values - * with unsigned longs. */ - max_pull = min(sds->max_load - sds->avg_load, load_above_capacity); + max_pull = min(busiest->avg_load - sds->avg_load, load_above_capacity); /* How much load to actually move to equalise the imbalance */ - env->imbalance = min(max_pull * sds->busiest->sgp->power, - (sds->avg_load - sds->this_load) * sds->this->sgp->power) - / SCHED_POWER_SCALE; + env->imbalance = min( + max_pull * busiest->group_capacity, + (sds->avg_load - local->avg_load) * local->group_capacity + ) / SCHED_CAPACITY_SCALE; + + /* Boost imbalance to allow misfit task to be balanced. */ + if (busiest->group_type == group_misfit_task) + env->imbalance = max_t(long, env->imbalance, + busiest->group_misfit_task); /* * if *imbalance is less than the average load per runnable task @@ -7313,9 +7562,8 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s * a think about bumping its value to force at least one task to be * moved */ - if (env->imbalance < sds->busiest_load_per_task) + if (env->imbalance < busiest->load_per_task) return fix_small_imbalance(env, sds); - } /******* find_busiest_group() helpers end here *********************/ @@ -7331,158 +7579,108 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s * to restore balance. * * @env: The load balancing environment. - * @balance: Pointer to a variable indicating if this_cpu - * is the appropriate cpu to perform load balancing at this_level. * - * Returns: - the busiest group if imbalance exists. + * Return: - The busiest group if imbalance exists. * - If no imbalance and user has opted for power-savings balance, * return the least loaded group whose CPUs can be * put to idle by rebalancing its tasks onto our group. */ -static struct sched_group * -find_busiest_group(struct lb_env *env, int *balance) +static struct sched_group *find_busiest_group(struct lb_env *env) { + struct sg_lb_stats *local, *busiest; struct sd_lb_stats sds; - memset(&sds, 0, sizeof(sds)); + init_sd_lb_stats(&sds); /* * Compute the various statistics relavent for load balancing at * this level. */ - update_sd_lb_stats(env, balance, &sds); + update_sd_lb_stats(env, &sds); - /* - * this_cpu is not the appropriate cpu to perform load balancing at - * this level. - */ - if (!(*balance)) - goto ret; + if (energy_aware() && !env->dst_rq->rd->overutilized) + goto out_balanced; + local = &sds.local_stat; + busiest = &sds.busiest_stat; + + /* ASYM feature bypasses nice load balance check */ if ((env->idle == CPU_IDLE || env->idle == CPU_NEWLY_IDLE) && check_asym_packing(env, &sds)) return sds.busiest; /* There is no busy sibling group to pull tasks from */ - if (!sds.busiest || sds.busiest_nr_running == 0) - goto out_balanced; - - if (env->flags & LBF_HMP_ACTIVE_BALANCE) - goto force_balance; - - if (bail_inter_cluster_balance(env, &sds)) + if (!sds.busiest || busiest->sum_nr_running == 0) goto out_balanced; - sds.avg_load = (SCHED_POWER_SCALE * sds.total_load) / sds.total_pwr; + sds.avg_load = (SCHED_CAPACITY_SCALE * sds.total_load) + / sds.total_capacity; /* * If the busiest group is imbalanced the below checks don't - * work because they assumes all things are equal, which typically + * work because they assume all things are equal, which typically * isn't true due to cpus_allowed constraints and the like. */ - if (sds.group_imb) + if (busiest->group_type == group_imbalanced) goto force_balance; /* SD_BALANCE_NEWIDLE trumps SMP nice when underutilized */ - if (env->idle == CPU_NEWLY_IDLE && sds.this_has_capacity && - !sds.busiest_has_capacity) + if (env->idle == CPU_NEWLY_IDLE && group_has_capacity(env, local) && + busiest->group_no_capacity) + goto force_balance; + + /* Misfitting tasks should be dealt with regardless of the avg load */ + if (busiest->group_type == group_misfit_task) { goto force_balance; + } /* - * If the local group is more busy than the selected busiest group + * If the local group is busier than the selected busiest group * don't try and pull any tasks. */ - if (sds.this_load >= sds.max_load) + if (local->avg_load >= busiest->avg_load) goto out_balanced; /* * Don't pull any tasks if this group is already above the domain * average load. */ - if (sds.this_load >= sds.avg_load) + if (local->avg_load >= sds.avg_load) goto out_balanced; if (env->idle == CPU_IDLE) { /* - * This cpu is idle. If the busiest group load doesn't - * have more tasks than the number of available cpu's and - * there is no imbalance between this and busiest group - * wrt to idle cpu's, it is balanced. + * This cpu is idle. If the busiest group is not overloaded + * and there is no imbalance between this and busiest group + * wrt idle cpus, it is balanced. The imbalance becomes + * significant if the diff is greater than 1 otherwise we + * might end up to just move the imbalance on another group */ - if ((sds.this_idle_cpus <= sds.busiest_idle_cpus + 1) && - sds.busiest_nr_running <= sds.busiest_group_weight) + if ((busiest->group_type != group_overloaded) && + (local->idle_cpus <= (busiest->idle_cpus + 1)) && + !group_smaller_cpu_capacity(sds.busiest, sds.local)) goto out_balanced; } else { /* * In the CPU_NEWLY_IDLE, CPU_NOT_IDLE cases, use * imbalance_pct to be conservative. */ - if (100 * sds.max_load <= env->sd->imbalance_pct * sds.this_load) + if (100 * busiest->avg_load <= + env->sd->imbalance_pct * local->avg_load) goto out_balanced; } force_balance: + env->busiest_group_type = busiest->group_type; /* Looks like there is an imbalance. Compute it */ calculate_imbalance(env, &sds); return sds.busiest; out_balanced: -ret: env->imbalance = 0; return NULL; } -#ifdef CONFIG_SCHED_HMP -static struct rq *find_busiest_queue_hmp(struct lb_env *env, - struct sched_group *group) -{ - struct rq *busiest = NULL, *busiest_big = NULL; - u64 max_runnable_avg = 0, max_runnable_avg_big = 0; - int max_nr_big = 0, nr_big; - bool find_big = !!(env->flags & LBF_BIG_TASK_ACTIVE_BALANCE); - int i; - - for_each_cpu(i, sched_group_cpus(group)) { - struct rq *rq = cpu_rq(i); - u64 cumulative_runnable_avg = - rq->hmp_stats.cumulative_runnable_avg; - - if (!cpumask_test_cpu(i, env->cpus)) - continue; - - - if (find_big) { - nr_big = nr_big_tasks(rq); - if (nr_big > max_nr_big || - (nr_big > 0 && nr_big == max_nr_big && - cumulative_runnable_avg > max_runnable_avg_big)) { - max_runnable_avg_big = cumulative_runnable_avg; - busiest_big = rq; - max_nr_big = nr_big; - continue; - } - } - - if (cumulative_runnable_avg > max_runnable_avg) { - max_runnable_avg = cumulative_runnable_avg; - busiest = rq; - } - } - - if (busiest_big) - return busiest_big; - - env->flags &= ~LBF_BIG_TASK_ACTIVE_BALANCE; - return busiest; -} -#else -static inline struct rq *find_busiest_queue_hmp(struct lb_env *env, - struct sched_group *group) -{ - return NULL; -} -#endif - /* * find_busiest_queue - find the busiest runqueue among the cpus in group. */ @@ -7490,44 +7688,65 @@ static struct rq *find_busiest_queue(struct lb_env *env, struct sched_group *group) { struct rq *busiest = NULL, *rq; - unsigned long max_load = 0; + unsigned long busiest_load = 0, busiest_capacity = 1; int i; - if (sched_enable_hmp) - return find_busiest_queue_hmp(env, group); - - for_each_cpu(i, sched_group_cpus(group)) { - unsigned long power = power_of(i); - unsigned long capacity = DIV_ROUND_CLOSEST(power, - SCHED_POWER_SCALE); - unsigned long wl; + for_each_cpu_and(i, sched_group_cpus(group), env->cpus) { + unsigned long capacity, wl; + enum fbq_type rt; - if (!capacity) - capacity = fix_small_capacity(env->sd, group); + rq = cpu_rq(i); + rt = fbq_classify_rq(rq); - if (!cpumask_test_cpu(i, env->cpus)) + /* + * We classify groups/runqueues into three groups: + * - regular: there are !numa tasks + * - remote: there are numa tasks that run on the 'wrong' node + * - all: there is no distinction + * + * In order to avoid migrating ideally placed numa tasks, + * ignore those when there's better options. + * + * If we ignore the actual busiest queue to migrate another + * task, the next balance pass can still reduce the busiest + * queue by moving tasks around inside the node. + * + * If we cannot move enough load due to this classification + * the next pass will adjust the group classification and + * allow migration of more tasks. + * + * Both cases only affect the total convergence complexity. + */ + if (rt > env->fbq_type) continue; - rq = cpu_rq(i); + capacity = capacity_of(i); + wl = weighted_cpuload(i); /* * When comparing with imbalance, use weighted_cpuload() - * which is not scaled with the cpu power. + * which is not scaled with the cpu capacity. */ - if (capacity && rq->nr_running == 1 && wl > env->imbalance) + if (rq->nr_running == 1 && wl > env->imbalance && + !check_cpu_capacity(rq, env->sd) && + env->busiest_group_type != group_misfit_task) continue; /* * For the load comparisons with the other cpu's, consider - * the weighted_cpuload() scaled with the cpu power, so that - * the load can be moved away from the cpu that is potentially - * running at a lower capacity. + * the weighted_cpuload() scaled with the cpu capacity, so + * that the load can be moved away from the cpu that is + * potentially running at a lower capacity. + * + * Thus we're looking for max(wl_i / capacity_i), crosswise + * multiplication to rid ourselves of the division works out + * to: wl_i * capacity_j > wl_j * capacity_i; where j is + * our previous maximum. */ - wl = (wl * SCHED_POWER_SCALE) / power; - - if (wl > max_load) { - max_load = wl; + if (wl * busiest_capacity > busiest_load * capacity) { + busiest_load = wl; + busiest_capacity = capacity; busiest = rq; } } @@ -7539,7 +7758,7 @@ static struct rq *find_busiest_queue(struct lb_env *env, * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but * so long as it is large enough. */ -#define MAX_PINNED_INTERVAL 16 +#define MAX_PINNED_INTERVAL 512 /* Working cpumask for load_balance and load_balance_newidle. */ DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); @@ -7548,9 +7767,6 @@ static int need_active_balance(struct lb_env *env) { struct sched_domain *sd = env->sd; - if (env->flags & LBF_HMP_ACTIVE_BALANCE) - return 1; - if (env->idle == CPU_NEWLY_IDLE) { /* @@ -7562,35 +7778,89 @@ static int need_active_balance(struct lb_env *env) return 1; } + /* + * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task. + * It's worth migrating the task if the src_cpu's capacity is reduced + * because of other sched_class or IRQs if more capacity stays + * available on dst_cpu. + */ + if ((env->idle != CPU_NOT_IDLE) && + (env->src_rq->cfs.h_nr_running == 1)) { + if ((check_cpu_capacity(env->src_rq, sd)) && + (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) + return 1; + } + if ((capacity_of(env->src_cpu) < capacity_of(env->dst_cpu)) && + env->src_rq->cfs.h_nr_running == 1 && + cpu_overutilized(env->src_cpu) && + !cpu_overutilized(env->dst_cpu)) { + return 1; + } + return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2); } +static int active_load_balance_cpu_stop(void *data); + +static int should_we_balance(struct lb_env *env) +{ + struct sched_group *sg = env->sd->groups; + struct cpumask *sg_cpus, *sg_mask; + int cpu, balance_cpu = -1; + + /* + * In the newly idle case, we will allow all the cpu's + * to do the newly idle load balance. + */ + if (env->idle == CPU_NEWLY_IDLE) + return 1; + + sg_cpus = sched_group_cpus(sg); + sg_mask = sched_group_mask(sg); + /* Try to find first idle cpu */ + for_each_cpu_and(cpu, sg_cpus, env->cpus) { + if (!cpumask_test_cpu(cpu, sg_mask) || !idle_cpu(cpu)) + continue; + + balance_cpu = cpu; + break; + } + + if (balance_cpu == -1) + balance_cpu = group_balance_cpu(sg); + + /* + * First idle cpu or the first cpu(busiest) in this sched group + * is eligible for doing load balancing at this and above domains. + */ + return balance_cpu == env->dst_cpu; +} + /* * Check this_cpu to ensure it is balanced within domain. Attempt to move * tasks if there is an imbalance. */ static int load_balance(int this_cpu, struct rq *this_rq, struct sched_domain *sd, enum cpu_idle_type idle, - int *balance) + int *continue_balancing) { int ld_moved, cur_ld_moved, active_balance = 0; + struct sched_domain *sd_parent = lb_sd_parent(sd) ? sd->parent : NULL; struct sched_group *group; - struct rq *busiest = NULL; + struct rq *busiest; unsigned long flags; - struct cpumask *cpus = __get_cpu_var(load_balance_mask); + struct cpumask *cpus = this_cpu_ptr(load_balance_mask); struct lb_env env = { - .sd = sd, - .dst_cpu = this_cpu, - .dst_rq = this_rq, - .dst_grpmask = sched_group_cpus(sd->groups), - .idle = idle, - .busiest_nr_running = 0, - .busiest_grp_capacity = 0, - .loop_break = sched_nr_migrate_break, - .cpus = cpus, - .flags = 0, - .loop = 0, + .sd = sd, + .dst_cpu = this_cpu, + .dst_rq = this_rq, + .dst_grpmask = sched_group_cpus(sd->groups), + .idle = idle, + .loop_break = sched_nr_migrate_break, + .cpus = cpus, + .fbq_type = all, + .tasks = LIST_HEAD_INIT(env.tasks), }; /* @@ -7602,15 +7872,15 @@ static int load_balance(int this_cpu, struct rq *this_rq, cpumask_copy(cpus, cpu_active_mask); - per_cpu(dbs_boost_load_moved, this_cpu) = 0; schedstat_inc(sd, lb_count[idle]); redo: - group = find_busiest_group(&env, balance); - - if (*balance == 0) + if (!should_we_balance(&env)) { + *continue_balancing = 0; goto out_balanced; + } + group = find_busiest_group(&env); if (!group) { schedstat_inc(sd, lb_nobusyg[idle]); goto out_balanced; @@ -7626,6 +7896,9 @@ static int load_balance(int this_cpu, struct rq *this_rq, schedstat_add(sd, lb_imbalance[idle], env.imbalance); + env.src_cpu = busiest->cpu; + env.src_rq = busiest; + ld_moved = 0; if (busiest->nr_running > 1) { /* @@ -7635,37 +7908,63 @@ static int load_balance(int this_cpu, struct rq *this_rq, * correctly treated as an imbalance. */ env.flags |= LBF_ALL_PINNED; - env.src_cpu = busiest->cpu; - env.src_rq = busiest; env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); - update_h_load(env.src_cpu); more_balance: - local_irq_save(flags); - double_rq_lock(env.dst_rq, busiest); - - /* The world might have changed. Validate assumptions */ - if (busiest->nr_running <= 1) { - double_rq_unlock(env.dst_rq, busiest); - local_irq_restore(flags); - env.flags &= ~LBF_ALL_PINNED; - goto no_move; - } + raw_spin_lock_irqsave(&busiest->lock, flags); /* * cur_ld_moved - load moved in current iteration * ld_moved - cumulative load moved across iterations */ - cur_ld_moved = move_tasks(&env); - ld_moved += cur_ld_moved; - double_rq_unlock(env.dst_rq, busiest); - local_irq_restore(flags); + cur_ld_moved = detach_tasks(&env); + /* + * We want to potentially update env.src_cpu's OPP. + * + * Add a margin (same ~20% used for the tipping point) + * to our request to provide some head room for the remaining + * tasks. + */ + if (sched_energy_freq() && cur_ld_moved) { + unsigned long req_cap = + get_boosted_cpu_usage(env.src_cpu); + + req_cap = req_cap * sysctl_sched_capacity_margin + >> SCHED_CAPACITY_SHIFT; + cpufreq_sched_set_cap(env.src_cpu, req_cap); + } /* - * some other cpu did the load balance for us. + * We've detached some tasks from busiest_rq. Every + * task is masked "TASK_ON_RQ_MIGRATING", so we can safely + * unlock busiest->lock, and we are able to be sure + * that nobody can manipulate the tasks in parallel. + * See task_rq_lock() family for the details. */ - if (cur_ld_moved && env.dst_cpu != smp_processor_id()) - resched_cpu(env.dst_cpu); + + raw_spin_unlock(&busiest->lock); + + if (cur_ld_moved) { + attach_tasks(&env); + ld_moved += cur_ld_moved; + /* + * We want to potentially update env.dst_cpu's OPP. + * + * Add a margin (same ~20% used for the tipping point) + * to our request to provide some head room if p's + * utilization further increases. + */ + if (sched_energy_freq()) { + unsigned long req_cap = + get_boosted_cpu_usage(env.dst_cpu); + + req_cap = req_cap * sysctl_sched_capacity_margin + >> SCHED_CAPACITY_SHIFT; + cpufreq_sched_set_cap(env.dst_cpu, req_cap); + } + } + + local_irq_restore(flags); if (env.flags & LBF_NEED_BREAK) { env.flags &= ~LBF_NEED_BREAK; @@ -7691,17 +7990,17 @@ static int load_balance(int this_cpu, struct rq *this_rq, * moreover subsequent load balance cycles should correct the * excess load moved. */ - if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) { + if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) { + + /* Prevent to re-select dst_cpu via env's cpus */ + cpumask_clear_cpu(env.dst_cpu, env.cpus); env.dst_rq = cpu_rq(env.new_dst_cpu); env.dst_cpu = env.new_dst_cpu; - env.flags &= ~LBF_SOME_PINNED; + env.flags &= ~LBF_DST_PINNED; env.loop = 0; env.loop_break = sched_nr_migrate_break; - /* Prevent to re-select dst_cpu via env's cpus */ - cpumask_clear_cpu(env.dst_cpu, env.cpus); - /* * Go back to "more_balance" rather than "redo" since we * need to continue with same src_cpu. @@ -7709,6 +8008,16 @@ static int load_balance(int this_cpu, struct rq *this_rq, goto more_balance; } + /* + * We failed to reach balance because of affinity. + */ + if (sd_parent) { + int *group_imbalance = &sd_parent->groups->sgc->imbalance; + + if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) + *group_imbalance = 1; + } + /* All tasks on this runqueue were pinned by CPU affinity */ if (unlikely(env.flags & LBF_ALL_PINNED)) { cpumask_clear_cpu(cpu_of(busiest), cpus); @@ -7717,24 +8026,21 @@ static int load_balance(int this_cpu, struct rq *this_rq, env.loop_break = sched_nr_migrate_break; goto redo; } - goto out_balanced; + goto out_all_pinned; } } -no_move: if (!ld_moved) { - if (!(env.flags & LBF_HMP_ACTIVE_BALANCE)) - schedstat_inc(sd, lb_failed[idle]); - + schedstat_inc(sd, lb_failed[idle]); /* * Increment the failure counter only on periodic balance. * We do not want newidle balance, which can be very * frequent, pollute the failure counter causing * excessive cache_hot migrations and active balances. */ - if (idle != CPU_NEWLY_IDLE && - !(env.flags & LBF_HMP_ACTIVE_BALANCE)) - sd->nr_balance_failed++; + if (idle != CPU_NEWLY_IDLE) + if (env.src_grp_nr_running > 1) + sd->nr_balance_failed++; if (need_active_balance(&env)) { raw_spin_lock_irqsave(&busiest->lock, flags); @@ -7767,7 +8073,6 @@ static int load_balance(int this_cpu, struct rq *this_rq, stop_one_cpu_nowait(cpu_of(busiest), active_load_balance_cpu_stop, busiest, &busiest->active_balance_work); - ld_moved++; } /* @@ -7776,29 +8081,9 @@ static int load_balance(int this_cpu, struct rq *this_rq, */ sd->nr_balance_failed = sd->cache_nice_tries+1; } - } else { - sd->nr_balance_failed = 0; - if (per_cpu(dbs_boost_needed, this_cpu)) { - struct migration_notify_data mnd; - - mnd.src_cpu = cpu_of(busiest); - mnd.dest_cpu = this_cpu; - mnd.load = per_cpu(dbs_boost_load_moved, this_cpu); - if (mnd.load > 100) - mnd.load = 100; - atomic_notifier_call_chain(&migration_notifier_head, - 0, (void *)&mnd); - per_cpu(dbs_boost_needed, this_cpu) = false; - per_cpu(dbs_boost_load_moved, this_cpu) = 0; - - } + } else + sd->nr_balance_failed = 0; - /* Assumes one 'busiest' cpu that we pulled tasks from */ - if (!same_freq_domain(this_cpu, cpu_of(busiest))) { - check_for_freq_change(this_rq); - check_for_freq_change(busiest); - } - } if (likely(!active_balance)) { /* We were unbalanced, so reset the balancing interval */ sd->balance_interval = sd->min_interval; @@ -7807,7 +8092,7 @@ static int load_balance(int this_cpu, struct rq *this_rq, * If we've begun active balancing, start to back off. This * case may not be covered by the all_pinned logic if there * is only 1 task on the busy runqueue (because we don't call - * move_tasks). + * detach_tasks). */ if (sd->balance_interval < sd->max_interval) sd->balance_interval *= 2; @@ -7816,6 +8101,23 @@ static int load_balance(int this_cpu, struct rq *this_rq, goto out; out_balanced: + /* + * We reach balance although we may have faced some affinity + * constraints. Clear the imbalance flag if it was set. + */ + if (sd_parent) { + int *group_imbalance = &sd_parent->groups->sgc->imbalance; + + if (*group_imbalance) + *group_imbalance = 0; + } + +out_all_pinned: + /* + * We reach balance because all tasks are pinned at this level so + * we can't migrate them. Let the imbalance flag set so parent level + * can try to migrate them. + */ schedstat_inc(sd, lb_balanced[idle]); sd->nr_balance_failed = 0; @@ -7829,97 +8131,140 @@ static int load_balance(int this_cpu, struct rq *this_rq, ld_moved = 0; out: - trace_sched_load_balance(this_cpu, idle, *balance, - group ? group->cpumask[0] : 0, - busiest ? busiest->nr_running : 0, - env.imbalance, env.flags, ld_moved, - sd->balance_interval); return ld_moved; } +static inline unsigned long +get_sd_balance_interval(struct sched_domain *sd, int cpu_busy) +{ + unsigned long interval = sd->balance_interval; + + if (cpu_busy) + interval *= sd->busy_factor; + + /* scale ms to jiffies */ + interval = msecs_to_jiffies(interval); + interval = clamp(interval, 1UL, max_load_balance_interval); + + return interval; +} + +static inline void +update_next_balance(struct sched_domain *sd, int cpu_busy, unsigned long *next_balance) +{ + unsigned long interval, next; + + interval = get_sd_balance_interval(sd, cpu_busy); + next = sd->last_balance + interval; + + if (time_after(*next_balance, next)) + *next_balance = next; +} + /* * idle_balance is called by schedule() if this_cpu is about to become * idle. Attempts to pull tasks from other CPUs. */ -void idle_balance(int this_cpu, struct rq *this_rq) +static int idle_balance(struct rq *this_rq) { + unsigned long next_balance = jiffies + HZ; + int this_cpu = this_rq->cpu; struct sched_domain *sd; int pulled_task = 0; - unsigned long next_balance = jiffies + HZ; - int i, cost; - int min_power = INT_MAX; - int balance_cpu = -1; - struct rq *balance_rq = NULL; + u64 curr_cost = 0; - this_rq->idle_stamp = this_rq->clock; + idle_enter_fair(this_rq); - if (this_rq->avg_idle < sysctl_sched_migration_cost) - return; + /* + * We must set idle_stamp _before_ calling idle_balance(), such that we + * measure the duration of idle_balance() as idle time. + */ + this_rq->idle_stamp = rq_clock(this_rq); - /* If this CPU is not the most power-efficient idle CPU in the - * lowest level domain, run load balance on behalf of that - * most power-efficient idle CPU. */ - rcu_read_lock(); - sd = rcu_dereference(per_cpu(sd_llc, this_cpu)); - if (sd && sysctl_sched_enable_power_aware) { - for_each_cpu(i, sched_domain_span(sd)) { - if (i == this_cpu || idle_cpu(i)) { - cost = power_cost_at_freq(i, 0); - if (cost < min_power) { - min_power = cost; - balance_cpu = i; - } - } - } - BUG_ON(balance_cpu == -1); + if ((!energy_aware() && (this_rq->avg_idle < sysctl_sched_migration_cost + || !this_rq->rd->overload)) || + (energy_aware() && !this_rq->rd->overutilized)) { + rcu_read_lock(); + sd = rcu_dereference_check_sched_domain(this_rq->sd); + if (sd) + update_next_balance(sd, 0, &next_balance); + rcu_read_unlock(); - } else { - balance_cpu = this_cpu; + goto out; } - rcu_read_unlock(); - balance_rq = cpu_rq(balance_cpu); - /* * Drop the rq->lock, but keep IRQ/preempt disabled. */ raw_spin_unlock(&this_rq->lock); - update_blocked_averages(balance_cpu); + update_blocked_averages(this_cpu); rcu_read_lock(); - for_each_domain(balance_cpu, sd) { - unsigned long interval; - int balance = 1; + for_each_domain(this_cpu, sd) { + int continue_balancing = 1; + u64 t0, domain_cost; if (!(sd->flags & SD_LOAD_BALANCE)) continue; + if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) { + update_next_balance(sd, 0, &next_balance); + break; + } + if (sd->flags & SD_BALANCE_NEWIDLE) { - /* If we've pulled tasks over stop searching: */ - pulled_task = load_balance(balance_cpu, balance_rq, - sd, CPU_NEWLY_IDLE, &balance); + t0 = sched_clock_cpu(this_cpu); + + pulled_task = load_balance(this_cpu, this_rq, + sd, CPU_NEWLY_IDLE, + &continue_balancing); + + domain_cost = sched_clock_cpu(this_cpu) - t0; + if (domain_cost > sd->max_newidle_lb_cost) + sd->max_newidle_lb_cost = domain_cost; + + curr_cost += domain_cost; } - interval = msecs_to_jiffies(sd->balance_interval); - if (time_after(next_balance, sd->last_balance + interval)) - next_balance = sd->last_balance + interval; - if (pulled_task) { - balance_rq->idle_stamp = 0; + update_next_balance(sd, 0, &next_balance); + + /* + * Stop searching for tasks to pull if there are + * now runnable tasks on this rq. + */ + if (pulled_task || this_rq->nr_running > 0) break; - } } rcu_read_unlock(); raw_spin_lock(&this_rq->lock); - if (balance_cpu == this_cpu && - (!pulled_task || time_after(jiffies, this_rq->next_balance))) { - /* - * We are going idle. next_balance may be set based on - * a busy processor. So reset next_balance. - */ + if (curr_cost > this_rq->max_idle_balance_cost) + this_rq->max_idle_balance_cost = curr_cost; + + /* + * While browsing the domains, we released the rq lock, a task could + * have been enqueued in the meantime. Since we're not going idle, + * pretend we pulled a task. + */ + if (this_rq->cfs.h_nr_running && !pulled_task) + pulled_task = 1; + +out: + /* Move the next balance forward */ + if (time_after(this_rq->next_balance, next_balance)) this_rq->next_balance = next_balance; + + /* Is there a task of a high priority class? */ + if (this_rq->nr_running != this_rq->cfs.h_nr_running) + pulled_task = -1; + + if (pulled_task) { + idle_exit_fair(this_rq); + this_rq->idle_stamp = 0; } + + return pulled_task; } /* @@ -7934,26 +8279,11 @@ static int active_load_balance_cpu_stop(void *data) int busiest_cpu = cpu_of(busiest_rq); int target_cpu = busiest_rq->push_cpu; struct rq *target_rq = cpu_rq(target_cpu); - struct sched_domain *sd = NULL; - struct task_struct *push_task; - struct lb_env env = { - .sd = sd, - .dst_cpu = target_cpu, - .dst_rq = target_rq, - .src_cpu = busiest_rq->cpu, - .src_rq = busiest_rq, - .idle = CPU_IDLE, - .busiest_nr_running = 0, - .busiest_grp_capacity = 0, - .flags = 0, - .loop = 0, - }; - bool moved = false; + struct sched_domain *sd; + struct task_struct *p = NULL; raw_spin_lock_irq(&busiest_rq->lock); - per_cpu(dbs_boost_load_moved, target_cpu) = 0; - /* make sure the requested cpu hasn't gone down in the meantime */ if (unlikely(busiest_cpu != smp_processor_id() || !busiest_rq->active_balance)) @@ -7970,21 +8300,6 @@ static int active_load_balance_cpu_stop(void *data) */ BUG_ON(busiest_rq == target_rq); - /* move a task from busiest_rq to target_rq */ - double_lock_balance(busiest_rq, target_rq); - - push_task = busiest_rq->push_task; - target_cpu = busiest_rq->push_cpu; - if (push_task) { - if (push_task->on_rq && push_task->state == TASK_RUNNING && - task_cpu(push_task) == busiest_cpu && - cpu_online(target_cpu)) { - move_task(push_task, &env); - moved = true; - } - goto out_unlock_balance; - } - /* Search for an sd spanning us and the target CPU. */ rcu_read_lock(); for_each_domain(target_cpu, sd) { @@ -7994,52 +8309,73 @@ static int active_load_balance_cpu_stop(void *data) } if (likely(sd)) { - env.sd = sd; + struct lb_env env = { + .sd = sd, + .dst_cpu = target_cpu, + .dst_rq = target_rq, + .src_cpu = busiest_rq->cpu, + .src_rq = busiest_rq, + .idle = CPU_IDLE, + }; + schedstat_inc(sd, alb_count); - if (move_one_task(&env)) { + p = detach_one_task(&env); + if (p) { schedstat_inc(sd, alb_pushed); - moved = true; - } else { - schedstat_inc(sd, alb_failed); + /* + * We want to potentially update env.src_cpu's OPP. + * + * Add a margin (same ~20% used for the tipping point) + * to our request to provide some head room for the + * remaining task. + */ + if (sched_energy_freq()) { + unsigned long req_cap = + get_boosted_cpu_usage(env.src_cpu); + + req_cap = req_cap * sysctl_sched_capacity_margin + >> SCHED_CAPACITY_SHIFT; + cpufreq_sched_set_cap(env.src_cpu, req_cap); + } } + else + schedstat_inc(sd, alb_failed); } rcu_read_unlock(); -out_unlock_balance: - double_unlock_balance(busiest_rq, target_rq); out_unlock: busiest_rq->active_balance = 0; - push_task = busiest_rq->push_task; - target_cpu = busiest_rq->push_cpu; - if (push_task) { - put_task_struct(push_task); - clear_reserved(target_cpu); - busiest_rq->push_task = NULL; - } - raw_spin_unlock_irq(&busiest_rq->lock); + raw_spin_unlock(&busiest_rq->lock); - if (moved && !same_freq_domain(busiest_cpu, target_cpu)) { - check_for_freq_change(busiest_rq); - check_for_freq_change(target_rq); - } + if (p) { + attach_one_task(target_rq, p); + /* + * We want to potentially update target_cpu's OPP. + * + * Add a margin (same ~20% used for the tipping point) + * to our request to provide some head room if p's utilization + * further increases. + */ + if (sched_energy_freq()) { + unsigned long req_cap = + get_boosted_cpu_usage(target_cpu); - if (per_cpu(dbs_boost_needed, target_cpu)) { - struct migration_notify_data mnd; + req_cap = req_cap * sysctl_sched_capacity_margin + >> SCHED_CAPACITY_SHIFT; + cpufreq_sched_set_cap(target_cpu, req_cap); + } + } - mnd.src_cpu = cpu_of(busiest_rq); - mnd.dest_cpu = target_cpu; - mnd.load = per_cpu(dbs_boost_load_moved, target_cpu); - if (mnd.load > 100) - mnd.load = 100; - atomic_notifier_call_chain(&migration_notifier_head, - 0, (void *)&mnd); + local_irq_enable(); - per_cpu(dbs_boost_needed, target_cpu) = false; - per_cpu(dbs_boost_load_moved, target_cpu) = 0; - } return 0; } +static inline int on_null_domain(struct rq *rq) +{ + return unlikely(!rcu_dereference_sched(rq->sd)); +} + #ifdef CONFIG_NO_HZ_COMMON /* * idle load balancing details @@ -8053,14 +8389,9 @@ static struct { unsigned long next_balance; /* in jiffy units */ } nohz ____cacheline_aligned; -static inline int find_new_ilb(int call_cpu, int type) +static inline int find_new_ilb(void) { - int ilb; - - if (sched_enable_hmp) - return find_new_hmp_ilb(call_cpu, type); - - ilb = cpumask_first(nohz.idle_cpus_mask); + int ilb = cpumask_first(nohz.idle_cpus_mask); if (ilb < nr_cpu_ids && idle_cpu(ilb)) return ilb; @@ -8073,13 +8404,13 @@ static inline int find_new_ilb(int call_cpu, int type) * nohz_load_balancer CPU (if there is one) otherwise fallback to any idle * CPU (if there is one). */ -static void nohz_balancer_kick(int cpu, int type) +static void nohz_balancer_kick(void) { int ilb_cpu; nohz.next_balance++; - ilb_cpu = find_new_ilb(cpu, type); + ilb_cpu = find_new_ilb(); if (ilb_cpu >= nr_cpu_ids) return; @@ -8099,8 +8430,13 @@ static void nohz_balancer_kick(int cpu, int type) static inline void nohz_balance_exit_idle(int cpu) { if (unlikely(test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))) { - cpumask_clear_cpu(cpu, nohz.idle_cpus_mask); - atomic_dec(&nohz.nr_cpus); + /* + * Completely isolated CPUs don't ever set, so we must test. + */ + if (likely(cpumask_test_cpu(cpu, nohz.idle_cpus_mask))) { + cpumask_clear_cpu(cpu, nohz.idle_cpus_mask); + atomic_dec(&nohz.nr_cpus); + } clear_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)); } } @@ -8111,14 +8447,13 @@ static inline void set_cpu_sd_state_busy(void) int cpu = smp_processor_id(); rcu_read_lock(); - sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); + sd = rcu_dereference(per_cpu(sd_busy, cpu)); if (!sd || !sd->nohz_idle) goto unlock; sd->nohz_idle = 0; - for (; sd; sd = sd->parent) - atomic_inc(&sd->groups->sgp->nr_busy_cpus); + atomic_inc(&sd->groups->sgc->nr_busy_cpus); unlock: rcu_read_unlock(); } @@ -8129,14 +8464,13 @@ void set_cpu_sd_state_idle(void) int cpu = smp_processor_id(); rcu_read_lock(); - sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); + sd = rcu_dereference(per_cpu(sd_busy, cpu)); if (!sd || sd->nohz_idle) goto unlock; sd->nohz_idle = 1; - for (; sd; sd = sd->parent) - atomic_dec(&sd->groups->sgp->nr_busy_cpus); + atomic_dec(&sd->groups->sgc->nr_busy_cpus); unlock: rcu_read_unlock(); } @@ -8156,12 +8490,18 @@ void nohz_balance_enter_idle(int cpu) if (test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu))) return; + /* + * If we're a completely isolated CPU, we don't play. + */ + if (on_null_domain(cpu_rq(cpu))) + return; + cpumask_set_cpu(cpu, nohz.idle_cpus_mask); atomic_inc(&nohz.nr_cpus); set_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)); } -static int __cpuinit sched_ilb_notifier(struct notifier_block *nfb, +static int sched_ilb_notifier(struct notifier_block *nfb, unsigned long action, void *hcpu) { switch (action & ~CPU_TASKS_FROZEN) { @@ -8191,49 +8531,67 @@ void update_max_interval(void) * * Balancing parameters are set up in init_sched_domains. */ -static void rebalance_domains(int cpu, enum cpu_idle_type idle) +static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle) { - int balance = 1; - struct rq *rq = cpu_rq(cpu); + int continue_balancing = 1; + int cpu = rq->cpu; unsigned long interval; struct sched_domain *sd; /* Earliest time when we have to do rebalance again */ unsigned long next_balance = jiffies + 60*HZ; int update_next_balance = 0; - int need_serialize; + int need_serialize, need_decay = 0; + u64 max_cost = 0; update_blocked_averages(cpu); rcu_read_lock(); for_each_domain(cpu, sd) { + /* + * Decay the newidle max times here because this is a regular + * visit to all the domains. Decay ~1% per second. + */ + if (time_after(jiffies, sd->next_decay_max_lb_cost)) { + sd->max_newidle_lb_cost = + (sd->max_newidle_lb_cost * 253) / 256; + sd->next_decay_max_lb_cost = jiffies + HZ; + need_decay = 1; + } + max_cost += sd->max_newidle_lb_cost; + if (!(sd->flags & SD_LOAD_BALANCE)) continue; - interval = sd->balance_interval; - if (idle != CPU_IDLE) - interval *= sd->busy_factor; + /* + * Stop the load balance at this level. There is another + * CPU in our sched group which is doing load balancing more + * actively. + */ + if (!continue_balancing) { + if (need_decay) + continue; + break; + } - /* scale ms to jiffies */ - interval = msecs_to_jiffies(interval); - interval = clamp(interval, 1UL, max_load_balance_interval); + interval = get_sd_balance_interval(sd, idle != CPU_IDLE); need_serialize = sd->flags & SD_SERIALIZE; - if (need_serialize) { if (!spin_trylock(&balancing)) goto out; } if (time_after_eq(jiffies, sd->last_balance + interval)) { - if (load_balance(cpu, rq, sd, idle, &balance)) { + if (load_balance(cpu, rq, sd, idle, &continue_balancing)) { /* - * The LBF_SOME_PINNED logic could have changed + * The LBF_DST_PINNED logic could have changed * env->dst_cpu, so we can't know our idle * state even if we migrated tasks. Update it. */ idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE; } sd->last_balance = jiffies; + interval = get_sd_balance_interval(sd, idle != CPU_IDLE); } if (need_serialize) spin_unlock(&balancing); @@ -8242,14 +8600,14 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle) next_balance = sd->last_balance + interval; update_next_balance = 1; } - + } + if (need_decay) { /* - * Stop the load balance at this level. There is another - * CPU in our sched group which is doing load balancing more - * actively. + * Ensure the rq-wide value also decays but keep it at a + * reasonable floor to avoid funnies with rq->avg_idle. */ - if (!balance) - break; + rq->max_idle_balance_cost = + max((u64)sysctl_sched_migration_cost, max_cost); } rcu_read_unlock(); @@ -8263,49 +8621,21 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle) } #ifdef CONFIG_NO_HZ_COMMON - -static int select_lowest_power_cpu(struct cpumask *cpus) -{ - int i, cost; - int lowest_power_cpu = -1; - int lowest_power = INT_MAX; - - if (sysctl_sched_enable_power_aware) { - for_each_cpu(i, cpus) { - cost = power_cost_at_freq(i, 0); - if (cost < lowest_power) { - lowest_power_cpu = i; - lowest_power = cost; - } - } - BUG_ON(lowest_power_cpu == -1); - return lowest_power_cpu; - } else { - return cpumask_first(cpus); - } -} - /* * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the * rebalancing for all the cpus for whom scheduler ticks are stopped. */ -static void nohz_idle_balance(int this_cpu, enum cpu_idle_type idle) +static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { - struct rq *this_rq = cpu_rq(this_cpu); + int this_cpu = this_rq->cpu; struct rq *rq; int balance_cpu; - struct cpumask cpus_to_balance; if (idle != CPU_IDLE || !test_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu))) goto end; - cpumask_copy(&cpus_to_balance, nohz.idle_cpus_mask); - - while (!cpumask_empty(&cpus_to_balance)) { - balance_cpu = select_lowest_power_cpu(&cpus_to_balance); - - cpumask_clear_cpu(balance_cpu, &cpus_to_balance); + for_each_cpu(balance_cpu, nohz.idle_cpus_mask) { if (balance_cpu == this_cpu || !idle_cpu(balance_cpu)) continue; @@ -8319,12 +8649,17 @@ static void nohz_idle_balance(int this_cpu, enum cpu_idle_type idle) rq = cpu_rq(balance_cpu); - raw_spin_lock_irq(&rq->lock); - update_rq_clock(rq); - update_idle_cpu_load(rq); - raw_spin_unlock_irq(&rq->lock); - - rebalance_domains(balance_cpu, CPU_IDLE); + /* + * If time for next balance is due, + * do the balance. + */ + if (time_after_eq(jiffies, rq->next_balance)) { + raw_spin_lock_irq(&rq->lock); + update_rq_clock(rq); + update_idle_cpu_load(rq); + raw_spin_unlock_irq(&rq->lock); + rebalance_domains(rq, CPU_IDLE); + } if (time_after(this_rq->next_balance, rq->next_balance)) this_rq->next_balance = rq->next_balance; @@ -8334,88 +8669,27 @@ static void nohz_idle_balance(int this_cpu, enum cpu_idle_type idle) clear_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu)); } -#ifdef CONFIG_SCHED_HMP -static inline int _nohz_kick_needed_hmp(struct rq *rq, int cpu, int *type) -{ - struct sched_domain *sd; - int i; - - if (rq->mostly_idle_freq && rq->cur_freq < rq->mostly_idle_freq - && rq->max_freq > rq->mostly_idle_freq) - return 0; - - if (rq->nr_running >= 2 && - (rq->nr_running - rq->hmp_stats.nr_small_tasks >= 2 || - rq->nr_running > rq->mostly_idle_nr_run || - cpu_load(cpu) > rq->mostly_idle_load)) { - - if (rq->capacity == max_capacity) - return 1; - - rcu_read_lock(); - sd = rcu_dereference_check_sched_domain(rq->sd); - if (!sd) { - rcu_read_unlock(); - return 0; - } - - for_each_cpu(i, sched_domain_span(sd)) { - if (cpu_load(i) < sched_spill_load) { - /* Change the kick type to limit to CPUs that - * are of equal or lower capacity. - */ - *type = NOHZ_KICK_RESTRICT; - break; - } - } - rcu_read_unlock(); - return 1; - } - - return 0; -} -#else -static inline int _nohz_kick_needed_hmp(struct rq *rq, int cpu, int *type) -{ - return 0; -} -#endif - -static inline int _nohz_kick_needed(struct rq *rq, int cpu, int *type) -{ - unsigned long now = jiffies; - - if (sched_enable_hmp) - return _nohz_kick_needed_hmp(rq, cpu, type); - - /* - * None are in tickless mode and hence no need for NOHZ idle load - * balancing. - */ - if (likely(!atomic_read(&nohz.nr_cpus))) - return 0; - - if (time_before(now, nohz.next_balance)) - return 0; - - return (rq->nr_running >= 2); -} - /* * Current heuristic for kicking the idle load balancer in the presence - * of an idle cpu is the system. + * of an idle cpu in the system. * - This rq has more than one task. - * - At any scheduler domain level, this cpu's scheduler group has multiple - * busy cpu's exceeding the group's power. + * - This rq has at least one CFS task and the capacity of the CPU is + * significantly reduced because of RT tasks or IRQs. + * - At parent of LLC scheduler domain level, this cpu's scheduler group has + * multiple busy cpu. * - For SD_ASYM_PACKING, if the lower numbered cpu's in the scheduler * domain span are idle. */ -static inline int nohz_kick_needed(struct rq *rq, int cpu, int *type) +static inline bool nohz_kick_needed(struct rq *rq) { + unsigned long now = jiffies; struct sched_domain *sd; + struct sched_group_capacity *sgc; + int nr_busy, cpu = rq->cpu; + bool kick = false; - if (unlikely(idle_cpu(cpu))) - return 0; + if (unlikely(rq->idle_balance)) + return false; /* * We may be recently in ticked or tickless idle mode. At the first @@ -8424,38 +8698,59 @@ static inline int nohz_kick_needed(struct rq *rq, int cpu, int *type) set_cpu_sd_state_busy(); nohz_balance_exit_idle(cpu); - if (_nohz_kick_needed(rq, cpu, type)) - goto need_kick; + /* + * None are in tickless mode and hence no need for NOHZ idle load + * balancing. + */ + if (likely(!atomic_read(&nohz.nr_cpus))) + return false; + + if (time_before(now, nohz.next_balance)) + return false; + + if (rq->nr_running >= 2 && + (!energy_aware() || cpu_overutilized(cpu))) + return true; + + /* Do idle load balance if there have misfit task */ + if (energy_aware() && rq->misfit_task) + return true; rcu_read_lock(); - for_each_domain(cpu, sd) { - struct sched_group *sg = sd->groups; - struct sched_group_power *sgp = sg->sgp; - int nr_busy = atomic_read(&sgp->nr_busy_cpus); + sd = rcu_dereference(per_cpu(sd_busy, cpu)); + if (sd && !energy_aware()) { + sgc = sd->groups->sgc; + nr_busy = atomic_read(&sgc->nr_busy_cpus); + + if (nr_busy > 1) { + kick = true; + goto unlock; + } -#ifndef CONFIG_SCHED_HMP - if (sd->flags & SD_SHARE_PKG_RESOURCES && nr_busy > 1) - goto need_kick_unlock; -#endif + } - if (sd->flags & SD_ASYM_PACKING && nr_busy != sg->group_weight - && (cpumask_first_and(nohz.idle_cpus_mask, - sched_domain_span(sd)) < cpu)) - goto need_kick_unlock; + sd = rcu_dereference(rq->sd); + if (sd) { + if ((rq->cfs.h_nr_running >= 1) && + check_cpu_capacity(rq, sd)) { + kick = true; + goto unlock; + } + } - if (!(sd->flags & (SD_SHARE_PKG_RESOURCES | SD_ASYM_PACKING))) - break; + sd = rcu_dereference(per_cpu(sd_asym, cpu)); + if (sd && (cpumask_first_and(nohz.idle_cpus_mask, + sched_domain_span(sd)) < cpu)) { + kick = true; + goto unlock; } - rcu_read_unlock(); - return 0; -need_kick_unlock: +unlock: rcu_read_unlock(); -need_kick: - return 1; + return kick; } #else -static void nohz_idle_balance(int this_cpu, enum cpu_idle_type idle) { } +static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { } #endif /* @@ -8464,40 +8759,36 @@ static void nohz_idle_balance(int this_cpu, enum cpu_idle_type idle) { } */ static void run_rebalance_domains(struct softirq_action *h) { - int this_cpu = smp_processor_id(); - struct rq *this_rq = cpu_rq(this_cpu); + struct rq *this_rq = this_rq(); enum cpu_idle_type idle = this_rq->idle_balance ? CPU_IDLE : CPU_NOT_IDLE; - rebalance_domains(this_cpu, idle); - /* * If this cpu has a pending nohz_balance_kick, then do the * balancing on behalf of the other idle cpus whose ticks are - * stopped. + * stopped. Do nohz_idle_balance *before* rebalance_domains to + * give the idle cpus a chance to load balance. Else we may + * load balance only within the local sched_domain hierarchy + * and abort nohz_idle_balance altogether if we pull some load. */ - nohz_idle_balance(this_cpu, idle); -} - -static inline int on_null_domain(int cpu) -{ - return !rcu_dereference_sched(cpu_rq(cpu)->sd); + nohz_idle_balance(this_rq, idle); + rebalance_domains(this_rq, idle); } /* * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing. */ -void trigger_load_balance(struct rq *rq, int cpu) +void trigger_load_balance(struct rq *rq) { - int type = NOHZ_KICK_ANY; - /* Don't need to rebalance while attached to NULL domain */ - if (time_after_eq(jiffies, rq->next_balance) && - likely(!on_null_domain(cpu))) + if (unlikely(on_null_domain(rq))) + return; + + if (time_after_eq(jiffies, rq->next_balance)) raise_softirq(SCHED_SOFTIRQ); #ifdef CONFIG_NO_HZ_COMMON - if (nohz_kick_needed(rq, cpu, &type) && likely(!on_null_domain(cpu))) - nohz_balancer_kick(cpu, type); + if (nohz_kick_needed(rq)) + nohz_balancer_kick(); #endif } @@ -8531,10 +8822,34 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) entity_tick(cfs_rq, se, queued); } - if (sched_feat_numa(NUMA)) + if (numabalancing_enabled) task_tick_numa(rq, curr); update_rq_runnable_avg(rq, 1); + + if (!rq->rd->overutilized && cpu_overutilized(task_cpu(curr))) + rq->rd->overutilized = true; + + rq->misfit_task = !task_fits_capacity(curr, rq->cpu); + + /* + * To make free room for a task that is building up its "real" + * utilization and to harm its performance the least, request a + * jump to max OPP as soon as get_cpu_usage() crosses the UP + * threshold. The UP threshold is built relative to the current + * capacity (OPP), by using same margin used to tell if a cpu + * is overutilized (capacity_margin). + */ + if (sched_energy_freq()) { + int cpu = cpu_of(rq); + unsigned long capacity_orig = capacity_orig_of(cpu); + unsigned long capacity_curr = capacity_curr_of(cpu); + + if (capacity_curr < capacity_orig && + (capacity_curr * SCHED_LOAD_SCALE) < + (get_cpu_usage(cpu) * sysctl_sched_capacity_margin)) + cpufreq_sched_set_cap(cpu, capacity_orig); + } } /* @@ -8579,7 +8894,7 @@ static void task_fork_fair(struct task_struct *p) * 'current' within the tree based on its new key value. */ swap(curr->vruntime, se->vruntime); - resched_task(rq->curr); + resched_curr(rq); } se->vruntime -= cfs_rq->min_vruntime; @@ -8594,7 +8909,7 @@ static void task_fork_fair(struct task_struct *p) static void prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio) { - if (!p->se.on_rq) + if (!task_on_rq_queued(p)) return; /* @@ -8604,7 +8919,7 @@ prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio) */ if (rq->curr == p) { if (p->prio > oldprio) - resched_task(rq->curr); + resched_curr(rq); } else check_preempt_curr(rq, p, 0); } @@ -8619,11 +8934,11 @@ static void switched_from_fair(struct rq *rq, struct task_struct *p) * switched back to the fair class the enqueue_entity(.flags=0) will * do the right thing. * - * If it's on_rq, then the dequeue_entity(.flags=0) will already - * have normalized the vruntime, if it's !on_rq, then only when + * If it's queued, then the dequeue_entity(.flags=0) will already + * have normalized the vruntime, if it's !queued, then only when * the task is sleeping will it still have non-normalized vruntime. */ - if (!p->on_rq && p->state != TASK_RUNNING) { + if (!task_on_rq_queued(p) && p->state != TASK_RUNNING) { /* * Fix up our vruntime so that the current sleep doesn't * cause 'unlimited' sleep bonus. @@ -8632,17 +8947,17 @@ static void switched_from_fair(struct rq *rq, struct task_struct *p) se->vruntime -= cfs_rq->min_vruntime; } -#if defined(CONFIG_FAIR_GROUP_SCHED) && defined(CONFIG_SMP) +#ifdef CONFIG_SMP /* * Remove our load from contribution when we leave sched_fair * and ensure we don't carry in an old decay_count if we * switch back. */ - if (p->se.avg.decay_count) { - struct cfs_rq *cfs_rq = cfs_rq_of(&p->se); - __synchronize_entity_decay(&p->se); - subtract_blocked_load_contrib(cfs_rq, - p->se.avg.load_avg_contrib); + if (se->avg.decay_count) { + __synchronize_entity_decay(se); + subtract_blocked_load_contrib(cfs_rq, se->avg.load_avg_contrib); + subtract_utilization_blocked_contrib(cfs_rq, + se->avg.utilization_avg_contrib); } #endif } @@ -8652,8 +8967,31 @@ static void switched_from_fair(struct rq *rq, struct task_struct *p) */ static void switched_to_fair(struct rq *rq, struct task_struct *p) { - if (!p->se.on_rq) + struct sched_entity *se = &p->se; + +#ifdef CONFIG_FAIR_GROUP_SCHED + /* + * Since the real-depth could have been changed (only FAIR + * class maintain depth value), reset depth properly. + */ + se->depth = se->parent ? se->parent->depth + 1 : 0; +#endif + + if (!task_on_rq_queued(p)) { + + /* + * Ensure the task has a non-normalized vruntime when it is switched + * back to the fair class with !queued, so that enqueue_entity() at + * wake-up time will do the right thing. + * + * If it's queued, then the enqueue_entity(.flags=0) makes the task + * has non-normalized vruntime, if it's !queued, then it still has + * normalized vruntime. + */ + if (p->state != TASK_RUNNING) + se->vruntime += cfs_rq_of(se)->min_vruntime; return; + } /* * We were most likely switched from sched_rt, so @@ -8661,7 +8999,7 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p) * if we can still preempt the current task. */ if (rq->curr == p) - resched_task(rq->curr); + resched_curr(rq); else check_preempt_curr(rq, p, 0); } @@ -8691,16 +9029,19 @@ void init_cfs_rq(struct cfs_rq *cfs_rq) #ifndef CONFIG_64BIT cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; #endif -#if defined(CONFIG_FAIR_GROUP_SCHED) && defined(CONFIG_SMP) +#ifdef CONFIG_SMP atomic64_set(&cfs_rq->decay_counter, 1); - atomic64_set(&cfs_rq->removed_load, 0); + atomic_long_set(&cfs_rq->removed_load, 0); + atomic_long_set(&cfs_rq->removed_utilization, 0); #endif } #ifdef CONFIG_FAIR_GROUP_SCHED -static void task_move_group_fair(struct task_struct *p, int on_rq) +static void task_move_group_fair(struct task_struct *p, int queued) { + struct sched_entity *se = &p->se; struct cfs_rq *cfs_rq; + /* * If the task was not on the rq at the time of this cgroup movement * it must have been asleep, sleeping tasks keep their ->vruntime @@ -8715,7 +9056,7 @@ static void task_move_group_fair(struct task_struct *p, int on_rq) * fair sleeper stuff for the first placement, but who cares. */ /* - * When !on_rq, vruntime of the task has usually NOT been normalized. + * When !queued, vruntime of the task has usually NOT been normalized. * But there are some cases where it has already been normalized: * * - Moving a forked child which is waiting for being woken up by @@ -8726,23 +9067,26 @@ static void task_move_group_fair(struct task_struct *p, int on_rq) * To prevent boost or penalty in the new cfs_rq caused by delta * min_vruntime between the two cfs_rqs, we skip vruntime adjustment. */ - if (!on_rq && (!p->se.sum_exec_runtime || p->state == TASK_WAKING)) - on_rq = 1; + if (!queued && (!se->sum_exec_runtime || p->state == TASK_WAKING)) + queued = 1; - if (!on_rq) - p->se.vruntime -= cfs_rq_of(&p->se)->min_vruntime; + if (!queued) + se->vruntime -= cfs_rq_of(se)->min_vruntime; set_task_rq(p, task_cpu(p)); - if (!on_rq) { - cfs_rq = cfs_rq_of(&p->se); - p->se.vruntime += cfs_rq->min_vruntime; + se->depth = se->parent ? se->parent->depth + 1 : 0; + if (!queued) { + cfs_rq = cfs_rq_of(se); + se->vruntime += cfs_rq->min_vruntime; #ifdef CONFIG_SMP /* * migrate_task_rq_fair() will have removed our previous * contribution, but we must synchronize for ongoing future * decay. */ - p->se.avg.decay_count = atomic64_read(&cfs_rq->decay_counter); - cfs_rq->blocked_load_avg += p->se.avg.load_avg_contrib; + se->avg.decay_count = atomic64_read(&cfs_rq->decay_counter); + cfs_rq->blocked_load_avg += se->avg.load_avg_contrib; + cfs_rq->utilization_blocked_avg += + se->avg.utilization_avg_contrib; #endif } } @@ -8838,10 +9182,13 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, if (!se) return; - if (!parent) + if (!parent) { se->cfs_rq = &rq->cfs; - else + se->depth = 0; + } else { se->cfs_rq = parent->my_q; + se->depth = parent->depth + 1; + } se->my_q = cfs_rq; /* guarantee group entities always have weight */ @@ -8876,6 +9223,9 @@ int sched_group_set_shares(struct task_group *tg, unsigned long shares) se = tg->se[i]; /* Propagate contribution to hierarchy */ raw_spin_lock_irqsave(&rq->lock, flags); + + /* Possible calls to update_curr() need rq clock */ + update_rq_clock(rq); for_each_sched_entity(se) update_cfs_shares(group_cfs_rq(se)); raw_spin_unlock_irqrestore(&rq->lock, flags); @@ -8931,9 +9281,8 @@ const struct sched_class fair_sched_class = { #ifdef CONFIG_SMP .select_task_rq = select_task_rq_fair, -#ifdef CONFIG_FAIR_GROUP_SCHED .migrate_task_rq = migrate_task_rq_fair, -#endif + .rq_online = rq_online_fair, .rq_offline = rq_offline_fair, @@ -8950,13 +9299,11 @@ const struct sched_class fair_sched_class = { .get_rr_interval = get_rr_interval_fair, + .update_curr = update_curr_fair, + #ifdef CONFIG_FAIR_GROUP_SCHED .task_move_group = task_move_group_fair, #endif -#ifdef CONFIG_SCHED_HMP - .inc_hmp_sched_stats = inc_hmp_sched_stats_fair, - .dec_hmp_sched_stats = dec_hmp_sched_stats_fair, -#endif }; #ifdef CONFIG_SCHED_DEBUG diff --git a/kernel/sched/features.h b/kernel/sched/features.h index cbb00d524aef4..eb53b51f374dd 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -36,19 +36,14 @@ SCHED_FEAT(CACHE_HOT_BUDDY, true) */ SCHED_FEAT(WAKEUP_PREEMPTION, true) -/* - * Use arch dependent cpu power functions - */ -SCHED_FEAT(ARCH_POWER, false) - SCHED_FEAT(HRTICK, false) SCHED_FEAT(DOUBLE_TICK, false) SCHED_FEAT(LB_BIAS, true) /* - * Decrement CPU power based on time not spent running tasks + * Decrement CPU capacity based on time not spent running tasks */ -SCHED_FEAT(NONTASK_POWER, true) +SCHED_FEAT(NONTASK_CAPACITY, true) /* * Queue remote wakeups on the target CPU and process them @@ -63,12 +58,29 @@ SCHED_FEAT(LB_MIN, false) /* * Apply the automatic NUMA scheduling policy. Enabled automatically * at runtime if running on a NUMA machine. Can be controlled via - * numa_balancing=. Allow PTE scanning to be forced on UMA machines - * for debugging the core machinery. + * numa_balancing= */ #ifdef CONFIG_NUMA_BALANCING SCHED_FEAT(NUMA, false) -SCHED_FEAT(NUMA_FORCE, false) + +/* + * NUMA_FAVOUR_HIGHER will favor moving tasks towards nodes where a + * higher number of hinting faults are recorded during active load + * balancing. + */ +SCHED_FEAT(NUMA_FAVOUR_HIGHER, true) + +/* + * NUMA_RESIST_LOWER will resist moving tasks towards nodes where a + * lower number of hinting faults have been recorded. As this has + * the potential to prevent a task ever migrating to a new node + * due to CPU overload it is disabled by default. + */ +SCHED_FEAT(NUMA_RESIST_LOWER, false) #endif -SCHED_FEAT(FORCE_CPU_THROTTLING_IMMINENT, false) +/* + * Energy aware scheduling. Use platform energy model to guide scheduling + * decisions optimizing for energy efficiency. + */ +SCHED_FEAT(ENERGY_AWARE, true) diff --git a/kernel/sched/idle_task.c b/kernel/sched/idle_task.c index d23d2964844e3..c65dac8c97cdd 100644 --- a/kernel/sched/idle_task.c +++ b/kernel/sched/idle_task.c @@ -9,37 +9,26 @@ #ifdef CONFIG_SMP static int -select_task_rq_idle(struct task_struct *p, int sd_flag, int flags) +select_task_rq_idle(struct task_struct *p, int cpu, int sd_flag, int flags) { return task_cpu(p); /* IDLE tasks as never migrated */ } - -static void pre_schedule_idle(struct rq *rq, struct task_struct *prev) -{ - idle_exit_fair(rq); - rq_last_tick_reset(rq); -} - -static void post_schedule_idle(struct rq *rq) -{ - idle_enter_fair(rq); -} #endif /* CONFIG_SMP */ + /* * Idle tasks are unconditionally rescheduled: */ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int flags) { - resched_task(rq->idle); + resched_curr(rq); } -static struct task_struct *pick_next_task_idle(struct rq *rq) +static struct task_struct * +pick_next_task_idle(struct rq *rq, struct task_struct *prev) { + put_prev_task(rq, prev); + schedstat_inc(rq, sched_goidle); -#ifdef CONFIG_SMP - /* Trigger the post schedule to do an idle_enter for CFS */ - rq->post_schedule = 1; -#endif return rq->idle; } @@ -58,6 +47,8 @@ dequeue_task_idle(struct rq *rq, struct task_struct *p, int flags) static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) { + idle_exit_fair(rq); + rq_last_tick_reset(rq); } static void task_tick_idle(struct rq *rq, struct task_struct *curr, int queued) @@ -84,20 +75,10 @@ static unsigned int get_rr_interval_idle(struct rq *rq, struct task_struct *task return 0; } -#ifdef CONFIG_SCHED_HMP - -static void -inc_hmp_sched_stats_idle(struct rq *rq, struct task_struct *p) +static void update_curr_idle(struct rq *rq) { } -static void -dec_hmp_sched_stats_idle(struct rq *rq, struct task_struct *p) -{ -} - -#endif - /* * Simple, special scheduling class for the per-CPU idle tasks: */ @@ -115,8 +96,6 @@ const struct sched_class idle_sched_class = { #ifdef CONFIG_SMP .select_task_rq = select_task_rq_idle, - .pre_schedule = pre_schedule_idle, - .post_schedule = post_schedule_idle, #endif .set_curr_task = set_curr_task_idle, @@ -126,8 +105,5 @@ const struct sched_class idle_sched_class = { .prio_changed = prio_changed_idle, .switched_to = switched_to_idle, -#ifdef CONFIG_SCHED_HMP - .inc_hmp_sched_stats = inc_hmp_sched_stats_idle, - .dec_hmp_sched_stats = dec_hmp_sched_stats_idle, -#endif + .update_curr = update_curr_idle, }; diff --git a/kernel/sched/loadavg.c b/kernel/sched/loadavg.c new file mode 100644 index 0000000000000..ef7159012cf36 --- /dev/null +++ b/kernel/sched/loadavg.c @@ -0,0 +1,394 @@ +/* + * kernel/sched/loadavg.c + * + * This file contains the magic bits required to compute the global loadavg + * figure. Its a silly number but people think its important. We go through + * great pains to make it work on big machines and tickless kernels. + */ + +#include + +#include "sched.h" + +/* + * Global load-average calculations + * + * We take a distributed and async approach to calculating the global load-avg + * in order to minimize overhead. + * + * The global load average is an exponentially decaying average of nr_running + + * nr_uninterruptible. + * + * Once every LOAD_FREQ: + * + * nr_active = 0; + * for_each_possible_cpu(cpu) + * nr_active += cpu_of(cpu)->nr_running + cpu_of(cpu)->nr_uninterruptible; + * + * avenrun[n] = avenrun[0] * exp_n + nr_active * (1 - exp_n) + * + * Due to a number of reasons the above turns in the mess below: + * + * - for_each_possible_cpu() is prohibitively expensive on machines with + * serious number of cpus, therefore we need to take a distributed approach + * to calculating nr_active. + * + * \Sum_i x_i(t) = \Sum_i x_i(t) - x_i(t_0) | x_i(t_0) := 0 + * = \Sum_i { \Sum_j=1 x_i(t_j) - x_i(t_j-1) } + * + * So assuming nr_active := 0 when we start out -- true per definition, we + * can simply take per-cpu deltas and fold those into a global accumulate + * to obtain the same result. See calc_load_fold_active(). + * + * Furthermore, in order to avoid synchronizing all per-cpu delta folding + * across the machine, we assume 10 ticks is sufficient time for every + * cpu to have completed this task. + * + * This places an upper-bound on the IRQ-off latency of the machine. Then + * again, being late doesn't loose the delta, just wrecks the sample. + * + * - cpu_rq()->nr_uninterruptible isn't accurately tracked per-cpu because + * this would add another cross-cpu cacheline miss and atomic operation + * to the wakeup path. Instead we increment on whatever cpu the task ran + * when it went into uninterruptible state and decrement on whatever cpu + * did the wakeup. This means that only the sum of nr_uninterruptible over + * all cpus yields the correct result. + * + * This covers the NO_HZ=n code, for extra head-aches, see the comment below. + */ + +/* Variables and functions for calc_load */ +atomic_long_t calc_load_tasks; +unsigned long calc_load_update; +unsigned long avenrun[3]; +EXPORT_SYMBOL(avenrun); /* should be removed */ + +/** + * get_avenrun - get the load average array + * @loads: pointer to dest load array + * @offset: offset to add + * @shift: shift count to shift the result left + * + * These values are estimates at best, so no need for locking. + */ +void get_avenrun(unsigned long *loads, unsigned long offset, int shift) +{ + loads[0] = (avenrun[0] + offset) << shift; + loads[1] = (avenrun[1] + offset) << shift; + loads[2] = (avenrun[2] + offset) << shift; +} + +long calc_load_fold_active(struct rq *this_rq) +{ + long nr_active, delta = 0; + + nr_active = this_rq->nr_running; + nr_active += (long)this_rq->nr_uninterruptible; + + if (nr_active != this_rq->calc_load_active) { + delta = nr_active - this_rq->calc_load_active; + this_rq->calc_load_active = nr_active; + } + + return delta; +} + +/* + * a1 = a0 * e + a * (1 - e) + */ +static unsigned long +calc_load(unsigned long load, unsigned long exp, unsigned long active) +{ + load *= exp; + load += active * (FIXED_1 - exp); + load += 1UL << (FSHIFT - 1); + return load >> FSHIFT; +} + +#ifdef CONFIG_NO_HZ_COMMON +/* + * Handle NO_HZ for the global load-average. + * + * Since the above described distributed algorithm to compute the global + * load-average relies on per-cpu sampling from the tick, it is affected by + * NO_HZ. + * + * The basic idea is to fold the nr_active delta into a global idle-delta upon + * entering NO_HZ state such that we can include this as an 'extra' cpu delta + * when we read the global state. + * + * Obviously reality has to ruin such a delightfully simple scheme: + * + * - When we go NO_HZ idle during the window, we can negate our sample + * contribution, causing under-accounting. + * + * We avoid this by keeping two idle-delta counters and flipping them + * when the window starts, thus separating old and new NO_HZ load. + * + * The only trick is the slight shift in index flip for read vs write. + * + * 0s 5s 10s 15s + * +10 +10 +10 +10 + * |-|-----------|-|-----------|-|-----------|-| + * r:0 0 1 1 0 0 1 1 0 + * w:0 1 1 0 0 1 1 0 0 + * + * This ensures we'll fold the old idle contribution in this window while + * accumlating the new one. + * + * - When we wake up from NO_HZ idle during the window, we push up our + * contribution, since we effectively move our sample point to a known + * busy state. + * + * This is solved by pushing the window forward, and thus skipping the + * sample, for this cpu (effectively using the idle-delta for this cpu which + * was in effect at the time the window opened). This also solves the issue + * of having to deal with a cpu having been in NOHZ idle for multiple + * LOAD_FREQ intervals. + * + * When making the ILB scale, we should try to pull this in as well. + */ +static atomic_long_t calc_load_idle[2]; +static int calc_load_idx; + +static inline int calc_load_write_idx(void) +{ + int idx = calc_load_idx; + + /* + * See calc_global_nohz(), if we observe the new index, we also + * need to observe the new update time. + */ + smp_rmb(); + + /* + * If the folding window started, make sure we start writing in the + * next idle-delta. + */ + if (!time_before(jiffies, calc_load_update)) + idx++; + + return idx & 1; +} + +static inline int calc_load_read_idx(void) +{ + return calc_load_idx & 1; +} + +void calc_load_enter_idle(void) +{ + struct rq *this_rq = this_rq(); + long delta; + + /* + * We're going into NOHZ mode, if there's any pending delta, fold it + * into the pending idle delta. + */ + delta = calc_load_fold_active(this_rq); + if (delta) { + int idx = calc_load_write_idx(); + + atomic_long_add(delta, &calc_load_idle[idx]); + } +} + +void calc_load_exit_idle(void) +{ + struct rq *this_rq = this_rq(); + + /* + * If we're still before the sample window, we're done. + */ + if (time_before(jiffies, this_rq->calc_load_update)) + return; + + /* + * We woke inside or after the sample window, this means we're already + * accounted through the nohz accounting, so skip the entire deal and + * sync up for the next window. + */ + this_rq->calc_load_update = calc_load_update; + if (time_before(jiffies, this_rq->calc_load_update + 10)) + this_rq->calc_load_update += LOAD_FREQ; +} + +static long calc_load_fold_idle(void) +{ + int idx = calc_load_read_idx(); + long delta = 0; + + if (atomic_long_read(&calc_load_idle[idx])) + delta = atomic_long_xchg(&calc_load_idle[idx], 0); + + return delta; +} + +/** + * fixed_power_int - compute: x^n, in O(log n) time + * + * @x: base of the power + * @frac_bits: fractional bits of @x + * @n: power to raise @x to. + * + * By exploiting the relation between the definition of the natural power + * function: x^n := x*x*...*x (x multiplied by itself for n times), and + * the binary encoding of numbers used by computers: n := \Sum n_i * 2^i, + * (where: n_i \elem {0, 1}, the binary vector representing n), + * we find: x^n := x^(\Sum n_i * 2^i) := \Prod x^(n_i * 2^i), which is + * of course trivially computable in O(log_2 n), the length of our binary + * vector. + */ +static unsigned long +fixed_power_int(unsigned long x, unsigned int frac_bits, unsigned int n) +{ + unsigned long result = 1UL << frac_bits; + + if (n) { + for (;;) { + if (n & 1) { + result *= x; + result += 1UL << (frac_bits - 1); + result >>= frac_bits; + } + n >>= 1; + if (!n) + break; + x *= x; + x += 1UL << (frac_bits - 1); + x >>= frac_bits; + } + } + + return result; +} + +/* + * a1 = a0 * e + a * (1 - e) + * + * a2 = a1 * e + a * (1 - e) + * = (a0 * e + a * (1 - e)) * e + a * (1 - e) + * = a0 * e^2 + a * (1 - e) * (1 + e) + * + * a3 = a2 * e + a * (1 - e) + * = (a0 * e^2 + a * (1 - e) * (1 + e)) * e + a * (1 - e) + * = a0 * e^3 + a * (1 - e) * (1 + e + e^2) + * + * ... + * + * an = a0 * e^n + a * (1 - e) * (1 + e + ... + e^n-1) [1] + * = a0 * e^n + a * (1 - e) * (1 - e^n)/(1 - e) + * = a0 * e^n + a * (1 - e^n) + * + * [1] application of the geometric series: + * + * n 1 - x^(n+1) + * S_n := \Sum x^i = ------------- + * i=0 1 - x + */ +static unsigned long +calc_load_n(unsigned long load, unsigned long exp, + unsigned long active, unsigned int n) +{ + return calc_load(load, fixed_power_int(exp, FSHIFT, n), active); +} + +/* + * NO_HZ can leave us missing all per-cpu ticks calling + * calc_load_account_active(), but since an idle CPU folds its delta into + * calc_load_tasks_idle per calc_load_account_idle(), all we need to do is fold + * in the pending idle delta if our idle period crossed a load cycle boundary. + * + * Once we've updated the global active value, we need to apply the exponential + * weights adjusted to the number of cycles missed. + */ +static void calc_global_nohz(void) +{ + long delta, active, n; + + if (!time_before(jiffies, calc_load_update + 10)) { + /* + * Catch-up, fold however many we are behind still + */ + delta = jiffies - calc_load_update - 10; + n = 1 + (delta / LOAD_FREQ); + + active = atomic_long_read(&calc_load_tasks); + active = active > 0 ? active * FIXED_1 : 0; + + avenrun[0] = calc_load_n(avenrun[0], EXP_1, active, n); + avenrun[1] = calc_load_n(avenrun[1], EXP_5, active, n); + avenrun[2] = calc_load_n(avenrun[2], EXP_15, active, n); + + calc_load_update += n * LOAD_FREQ; + } + + /* + * Flip the idle index... + * + * Make sure we first write the new time then flip the index, so that + * calc_load_write_idx() will see the new time when it reads the new + * index, this avoids a double flip messing things up. + */ + smp_wmb(); + calc_load_idx++; +} +#else /* !CONFIG_NO_HZ_COMMON */ + +static inline long calc_load_fold_idle(void) { return 0; } +static inline void calc_global_nohz(void) { } + +#endif /* CONFIG_NO_HZ_COMMON */ + +/* + * calc_load - update the avenrun load estimates 10 ticks after the + * CPUs have updated calc_load_tasks. + * + * Called from the global timer code. + */ +void calc_global_load(unsigned long ticks) +{ + long active, delta; + + if (time_before(jiffies, calc_load_update + 10)) + return; + + /* + * Fold the 'old' idle-delta to include all NO_HZ cpus. + */ + delta = calc_load_fold_idle(); + if (delta) + atomic_long_add(delta, &calc_load_tasks); + + active = atomic_long_read(&calc_load_tasks); + active = active > 0 ? active * FIXED_1 : 0; + + avenrun[0] = calc_load(avenrun[0], EXP_1, active); + avenrun[1] = calc_load(avenrun[1], EXP_5, active); + avenrun[2] = calc_load(avenrun[2], EXP_15, active); + + calc_load_update += LOAD_FREQ; + + /* + * In case we idled for multiple LOAD_FREQ intervals, catch up in bulk. + */ + calc_global_nohz(); +} + +/* + * Called from scheduler_tick() to periodically update this CPU's + * active count. + */ +void calc_global_load_tick(struct rq *this_rq) +{ + long delta; + + if (time_before(jiffies, this_rq->calc_load_update)) + return; + + delta = calc_load_fold_active(this_rq); + if (delta) + atomic_long_add(delta, &calc_load_tasks); + + this_rq->calc_load_update += LOAD_FREQ; +} diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 8b5840c5f1ab1..964211b5c794f 100755 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -6,8 +6,9 @@ #include "sched.h" #include -#include - +#ifdef CONFIG_MTPROF +#include "mt_sched_mon.h" +#endif int sched_rr_timeslice = RR_TIMESLICE; static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun); @@ -28,6 +29,11 @@ static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer) if (!overrun) break; +#ifdef CONFIG_MTPROF + /* mt throttle monitor */ + mt_rt_mon_switch(MON_RESET); + mt_rt_mon_switch(MON_START); +#endif idle = do_sched_rt_period_timer(rt_b, overrun); } @@ -80,6 +86,8 @@ void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq) rt_rq->overloaded = 0; plist_head_init(&rt_rq->pushable_tasks); #endif + /* We start is dequeued state, because no RT tasks are queued */ + rt_rq->rt_queued = 0; rt_rq->rt_time = 0; rt_rq->rt_throttled = 0; @@ -113,6 +121,13 @@ static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) return rt_se->rt_rq; } +static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) +{ + struct rt_rq *rt_rq = rt_se->rt_rq; + + return rt_rq->rq; +} + void free_rt_sched_group(struct task_group *tg) { int i; @@ -212,10 +227,16 @@ static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) return container_of(rt_rq, struct rq, rt); } -static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) +static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) { struct task_struct *p = rt_task_of(rt_se); - struct rq *rq = task_rq(p); + + return task_rq(p); +} + +static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) +{ + struct rq *rq = rq_of_rt_se(rt_se); return &rq->rt; } @@ -230,6 +251,14 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) #ifdef CONFIG_SMP +static int pull_rt_task(struct rq *this_rq); + +static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev) +{ + /* Try to pull RT tasks here if we lower this rq's prio */ + return rq->rt.highest_prio.curr > prev->prio; +} + static inline int rt_overloaded(struct rq *rq) { return atomic_read(&rq->rd->rto_count); @@ -247,8 +276,10 @@ static inline void rt_set_overload(struct rq *rq) * if we should look at the mask. It would be a shame * if we looked at the mask, but the mask was not * updated yet. + * + * Matched by the barrier in pull_rt_task(). */ - wmb(); + smp_wmb(); atomic_inc(&rq->rd->rto_count); } @@ -314,6 +345,15 @@ static inline int has_pushable_tasks(struct rq *rq) return !plist_head_empty(&rq->rt.pushable_tasks); } +static inline void set_post_schedule(struct rq *rq) +{ + /* + * We detect this state here so that we can avoid taking the RQ + * lock again later if there is no need to push + */ + rq->post_schedule = has_pushable_tasks(rq); +} + static void enqueue_pushable_task(struct rq *rq, struct task_struct *p) { plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); @@ -358,8 +398,24 @@ void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) { } +static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev) +{ + return false; +} + +static inline int pull_rt_task(struct rq *this_rq) +{ + return 0; +} + +static inline void set_post_schedule(struct rq *rq) +{ +} #endif /* CONFIG_SMP */ +static void enqueue_top_rt_rq(struct rt_rq *rt_rq); +static void dequeue_top_rt_rq(struct rt_rq *rt_rq); + static inline int on_rt_rq(struct sched_rt_entity *rt_se) { return !list_empty(&rt_se->run_list); @@ -400,20 +456,6 @@ static inline struct task_group *next_task_group(struct task_group *tg) (iter = next_task_group(iter)) && \ (rt_rq = iter->rt_rq[cpu_of(rq)]);) -static inline void list_add_leaf_rt_rq(struct rt_rq *rt_rq) -{ - list_add_rcu(&rt_rq->leaf_rt_rq_list, - &rq_of_rt_rq(rt_rq)->leaf_rt_rq_list); -} - -static inline void list_del_leaf_rt_rq(struct rt_rq *rt_rq) -{ - list_del_rcu(&rt_rq->leaf_rt_rq_list); -} - -#define for_each_leaf_rt_rq(rt_rq, rq) \ - list_for_each_entry_rcu(rt_rq, &rq->leaf_rt_rq_list, leaf_rt_rq_list) - #define for_each_sched_rt_entity(rt_se) \ for (; rt_se; rt_se = rt_se->parent) @@ -428,17 +470,21 @@ static void dequeue_rt_entity(struct sched_rt_entity *rt_se); static void sched_rt_rq_enqueue(struct rt_rq *rt_rq) { struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr; + struct rq *rq = rq_of_rt_rq(rt_rq); struct sched_rt_entity *rt_se; - int cpu = cpu_of(rq_of_rt_rq(rt_rq)); + int cpu = cpu_of(rq); rt_se = rt_rq->tg->rt_se[cpu]; if (rt_rq->rt_nr_running) { - if (rt_se && !on_rt_rq(rt_se)) + if (!rt_se) + enqueue_top_rt_rq(rt_rq); + else if (!on_rt_rq(rt_se)) enqueue_rt_entity(rt_se, false); + if (rt_rq->highest_prio.curr < curr->prio) - resched_task(curr); + resched_curr(rq); } } @@ -449,7 +495,9 @@ static void sched_rt_rq_dequeue(struct rt_rq *rt_rq) rt_se = rt_rq->tg->rt_se[cpu]; - if (rt_se && on_rt_rq(rt_se)) + if (!rt_se) + dequeue_top_rt_rq(rt_rq); + else if (on_rt_rq(rt_se)) dequeue_rt_entity(rt_se); } @@ -473,7 +521,7 @@ static int rt_se_boosted(struct sched_rt_entity *rt_se) #ifdef CONFIG_SMP static inline const struct cpumask *sched_rt_period_mask(void) { - return cpu_rq(smp_processor_id())->rd->span; + return this_rq()->rd->span; } #else static inline const struct cpumask *sched_rt_period_mask(void) @@ -510,17 +558,6 @@ typedef struct rt_rq *rt_rq_iter_t; #define for_each_rt_rq(rt_rq, iter, rq) \ for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL) -static inline void list_add_leaf_rt_rq(struct rt_rq *rt_rq) -{ -} - -static inline void list_del_leaf_rt_rq(struct rt_rq *rt_rq) -{ -} - -#define for_each_leaf_rt_rq(rt_rq, rq) \ - for (rt_rq = &rq->rt; rt_rq; rt_rq = NULL) - #define for_each_sched_rt_entity(rt_se) \ for (; rt_se; rt_se = NULL) @@ -531,12 +568,18 @@ static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq) { - if (rt_rq->rt_nr_running) - resched_task(rq_of_rt_rq(rt_rq)->curr); + struct rq *rq = rq_of_rt_rq(rt_rq); + + if (!rt_rq->rt_nr_running) + return; + + enqueue_top_rt_rq(rt_rq); + resched_curr(rq); } static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq) { + dequeue_top_rt_rq(rt_rq); } static inline int rt_rq_throttled(struct rt_rq *rt_rq) @@ -562,6 +605,14 @@ static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq) #endif /* CONFIG_RT_GROUP_SCHED */ +bool sched_rt_bandwidth_account(struct rt_rq *rt_rq) +{ + struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); + + return (hrtimer_active(&rt_b->rt_period_timer) || + rt_rq->rt_time < rt_b->rt_runtime); +} + #ifdef CONFIG_SMP /* * We ran out of runtime, see if we can borrow some from our neighbours. @@ -697,6 +748,9 @@ static void __disable_runtime(struct rq *rq) rt_rq->rt_throttled = 0; raw_spin_unlock(&rt_rq->rt_runtime_lock); raw_spin_unlock(&rt_b->rt_runtime_lock); + + /* Make rt_rq available for pick_next_task() */ + sched_rt_rq_enqueue(rt_rq); } } @@ -827,51 +881,6 @@ static inline int rt_se_prio(struct sched_rt_entity *rt_se) return rt_task_of(rt_se)->prio; } -static void dump_throttled_rt_tasks(struct rt_rq *rt_rq) -{ - struct rt_prio_array *array = &rt_rq->active; - struct sched_rt_entity *rt_se; - char buf[500]; - char *pos = buf; - char *end = buf + sizeof(buf); - int idx; - - pos += snprintf(pos, sizeof(buf), - "sched: RT throttling activated for rt_rq %p (cpu %d)\n", - rt_rq, cpu_of(rq_of_rt_rq(rt_rq))); - - if (bitmap_empty(array->bitmap, MAX_RT_PRIO)) - goto out; - - pos += snprintf(pos, end - pos, "potential CPU hogs:\n"); - idx = sched_find_first_bit(array->bitmap); - while (idx < MAX_RT_PRIO) { - list_for_each_entry(rt_se, array->queue + idx, run_list) { - struct task_struct *p; - - if (!rt_entity_is_task(rt_se)) - continue; - - p = rt_task_of(rt_se); - if (pos < end) - pos += snprintf(pos, end - pos, "\t%s (%d)\n", - p->comm, p->pid); - } - idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx + 1); - } -out: -#ifdef CONFIG_PANIC_ON_RT_THROTTLING - /* - * Use pr_err() in the BUG() case since printk_sched() will - * not get flushed and deadlock is not a concern. - */ - pr_err("%s", buf); - BUG(); -#else - printk_deferred("%s", buf); -#endif -} - static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) { u64 runtime = sched_rt_runtime(rt_rq); @@ -895,14 +904,15 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) * but accrue some time due to boosting. */ if (likely(rt_b->rt_runtime)) { - static bool once = false; - rt_rq->rt_throttled = 1; - - if (!once) { - once = true; - dump_throttled_rt_tasks(rt_rq); - } +#if 0 + printk_deferred_once("sched: RT throttling activated\n"); +#endif +#ifdef CONFIG_MTPROF + /* sched:rt throttle monitor */ + mt_rt_mon_switch(MON_STOP); + mt_rt_mon_print_task(); +#endif } else { /* * In case we did anyway, make it go away, @@ -929,13 +939,12 @@ static void update_curr_rt(struct rq *rq) { struct task_struct *curr = rq->curr; struct sched_rt_entity *rt_se = &curr->rt; - struct rt_rq *rt_rq = rt_rq_of_se(rt_se); u64 delta_exec; if (curr->sched_class != &rt_sched_class) return; - delta_exec = rq->clock_task - curr->se.exec_start; + delta_exec = rq_clock_task(rq) - curr->se.exec_start; if (unlikely((s64)delta_exec <= 0)) return; @@ -945,7 +954,7 @@ static void update_curr_rt(struct rq *rq) curr->se.sum_exec_runtime += delta_exec; account_group_exec_runtime(curr, delta_exec); - curr->se.exec_start = rq->clock_task; + curr->se.exec_start = rq_clock_task(rq); cpuacct_charge(curr, delta_exec); sched_rt_avg_update(rq, delta_exec); @@ -954,18 +963,50 @@ static void update_curr_rt(struct rq *rq) return; for_each_sched_rt_entity(rt_se) { - rt_rq = rt_rq_of_se(rt_se); + struct rt_rq *rt_rq = rt_rq_of_se(rt_se); if (sched_rt_runtime(rt_rq) != RUNTIME_INF) { raw_spin_lock(&rt_rq->rt_runtime_lock); rt_rq->rt_time += delta_exec; if (sched_rt_runtime_exceeded(rt_rq)) - resched_task(curr); + resched_curr(rq); raw_spin_unlock(&rt_rq->rt_runtime_lock); } } } +static void +dequeue_top_rt_rq(struct rt_rq *rt_rq) +{ + struct rq *rq = rq_of_rt_rq(rt_rq); + + BUG_ON(&rq->rt != rt_rq); + + if (!rt_rq->rt_queued) + return; + + BUG_ON(!rq->nr_running); + + sub_nr_running(rq, rt_rq->rt_nr_running); + rt_rq->rt_queued = 0; +} + +static void +enqueue_top_rt_rq(struct rt_rq *rt_rq) +{ + struct rq *rq = rq_of_rt_rq(rt_rq); + + BUG_ON(&rq->rt != rt_rq); + + if (rt_rq->rt_queued) + return; + if (rt_rq_throttled(rt_rq) || !rt_rq->rt_nr_running) + return; + + add_nr_running(rq, rt_rq->rt_nr_running); + rt_rq->rt_queued = 1; +} + #if defined CONFIG_SMP static void @@ -1088,37 +1129,24 @@ void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {} #endif /* CONFIG_RT_GROUP_SCHED */ -#ifdef CONFIG_SCHED_HMP - -static void -inc_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p) +static inline +unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se) { - inc_cumulative_runnable_avg(&rq->hmp_stats, p); -} + struct rt_rq *group_rq = group_rt_rq(rt_se); -static void -dec_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p) -{ - dec_cumulative_runnable_avg(&rq->hmp_stats, p); + if (group_rq) + return group_rq->rt_nr_running; + else + return 1; } -#else /* CONFIG_SCHED_HMP */ - -static inline void -inc_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p) { } - -static inline void -dec_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p) { } - -#endif /* CONFIG_SCHED_HMP */ - static inline void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) { int prio = rt_se_prio(rt_se); WARN_ON(!rt_prio(prio)); - rt_rq->rt_nr_running++; + rt_rq->rt_nr_running += rt_se_nr_running(rt_se); inc_rt_prio(rt_rq, prio); inc_rt_migration(rt_se, rt_rq); @@ -1130,7 +1158,7 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) { WARN_ON(!rt_prio(rt_se_prio(rt_se))); WARN_ON(!rt_rq->rt_nr_running); - rt_rq->rt_nr_running--; + rt_rq->rt_nr_running -= rt_se_nr_running(rt_se); dec_rt_prio(rt_rq, rt_se_prio(rt_se)); dec_rt_migration(rt_se, rt_rq); @@ -1153,9 +1181,6 @@ static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head) if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running)) return; - if (!rt_rq->rt_nr_running) - list_add_leaf_rt_rq(rt_rq); - if (head) list_add(&rt_se->run_list, queue); else @@ -1175,8 +1200,6 @@ static void __dequeue_rt_entity(struct sched_rt_entity *rt_se) __clear_bit(rt_se_prio(rt_se), array->bitmap); dec_rt_tasks(rt_se, rt_rq); - if (!rt_rq->rt_nr_running) - list_del_leaf_rt_rq(rt_rq); } /* @@ -1192,6 +1215,8 @@ static void dequeue_rt_stack(struct sched_rt_entity *rt_se) back = rt_se; } + dequeue_top_rt_rq(rt_rq_of_se(back)); + for (rt_se = back; rt_se; rt_se = rt_se->back) { if (on_rt_rq(rt_se)) __dequeue_rt_entity(rt_se); @@ -1200,13 +1225,18 @@ static void dequeue_rt_stack(struct sched_rt_entity *rt_se) static void enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head) { + struct rq *rq = rq_of_rt_se(rt_se); + dequeue_rt_stack(rt_se); for_each_sched_rt_entity(rt_se) __enqueue_rt_entity(rt_se, head); + enqueue_top_rt_rq(&rq->rt); } static void dequeue_rt_entity(struct sched_rt_entity *rt_se) { + struct rq *rq = rq_of_rt_se(rt_se); + dequeue_rt_stack(rt_se); for_each_sched_rt_entity(rt_se) { @@ -1215,6 +1245,7 @@ static void dequeue_rt_entity(struct sched_rt_entity *rt_se) if (rt_rq && rt_rq->rt_nr_running) __enqueue_rt_entity(rt_se, false); } + enqueue_top_rt_rq(&rq->rt); } /* @@ -1232,9 +1263,6 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags) if (!task_current(rq, p) && p->nr_cpus_allowed > 1) enqueue_pushable_task(rq, p); - - inc_nr_running(rq); - inc_hmp_sched_stats_rt(rq, p); } static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags) @@ -1245,9 +1273,6 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags) dequeue_rt_entity(rt_se); dequeue_pushable_task(rq, p); - - dec_nr_running(rq); - dec_hmp_sched_stats_rt(rq, p); } /* @@ -1288,36 +1313,14 @@ static void yield_task_rt(struct rq *rq) static int find_lowest_rq(struct task_struct *task); static int -select_task_rq_rt_hmp(struct task_struct *p, int sd_flag, int flags) -{ - int cpu, target; - - cpu = task_cpu(p); - - rcu_read_lock(); - target = find_lowest_rq(p); - if (target != -1) - cpu = target; - rcu_read_unlock(); - - return cpu; -} - -static int -select_task_rq_rt(struct task_struct *p, int sd_flag, int flags) +select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags) { struct task_struct *curr; struct rq *rq; - int cpu; - - cpu = task_cpu(p); if (p->nr_cpus_allowed == 1) goto out; - if (sched_enable_hmp) - return select_task_rq_rt_hmp(p, sd_flag, flags); - /* For anything but wake ups, just return the task_cpu */ if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK) goto out; @@ -1351,8 +1354,7 @@ select_task_rq_rt(struct task_struct *p, int sd_flag, int flags) */ if (curr && unlikely(rt_task(curr)) && (curr->nr_cpus_allowed < 2 || - curr->prio <= p->prio) && - (p->nr_cpus_allowed > 1)) { + curr->prio <= p->prio)) { int target = find_lowest_rq(p); if (target != -1) @@ -1382,7 +1384,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p) * to try and push current away: */ requeue_task_rt(rq, p, 1); - resched_task(rq->curr); + resched_curr(rq); } #endif /* CONFIG_SMP */ @@ -1393,7 +1395,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p) static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flags) { if (p->prio < rq->curr->prio) { - resched_task(rq->curr); + resched_curr(rq); return; } @@ -1436,15 +1438,7 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq) { struct sched_rt_entity *rt_se; struct task_struct *p; - struct rt_rq *rt_rq; - - rt_rq = &rq->rt; - - if (!rt_rq->rt_nr_running) - return NULL; - - if (rt_rq_throttled(rt_rq)) - return NULL; + struct rt_rq *rt_rq = &rq->rt; do { rt_se = pick_next_rt_entity(rq, rt_rq); @@ -1452,36 +1446,48 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq) rt_rq = group_rt_rq(rt_se); } while (rt_rq); - /* - * Force update of rq->clock_task in case we failed to do so in - * put_prev_task. A stale value can cause us to over-charge execution - * time to real-time task, that could trigger throttling unnecessarily - */ - if (rq->skip_clock_update > 0) { - rq->skip_clock_update = 0; - update_rq_clock(rq); - } p = rt_task_of(rt_se); - p->se.exec_start = rq->clock_task; + p->se.exec_start = rq_clock_task(rq); return p; } -static struct task_struct *pick_next_task_rt(struct rq *rq) +static struct task_struct * +pick_next_task_rt(struct rq *rq, struct task_struct *prev) { - struct task_struct *p = _pick_next_task_rt(rq); + struct task_struct *p; + struct rt_rq *rt_rq = &rq->rt; - /* The running task is never eligible for pushing */ - if (p) - dequeue_pushable_task(rq, p); + if (need_pull_rt_task(rq, prev)) { + pull_rt_task(rq); + /* + * pull_rt_task() can drop (and re-acquire) rq->lock; this + * means a dl or stop task can slip in, in which case we need + * to re-start task selection. + */ + if (unlikely((rq->stop && task_on_rq_queued(rq->stop)) || + rq->dl.dl_nr_running)) + return RETRY_TASK; + } -#ifdef CONFIG_SMP /* - * We detect this state here so that we can avoid taking the RQ - * lock again later if there is no need to push + * We may dequeue prev's rt_rq in put_prev_task(). + * So, we update time before rt_nr_running check. */ - rq->post_schedule = has_pushable_tasks(rq); -#endif + if (prev->sched_class == &rt_sched_class) + update_curr_rt(rq); + + if (!rt_rq->rt_queued) + return NULL; + + put_prev_task(rq, prev); + + p = _pick_next_task_rt(rq); + + /* The running task is never eligible for pushing */ + dequeue_pushable_task(rq, p); + + set_post_schedule(rq); return p; } @@ -1511,111 +1517,35 @@ static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu) return 0; } -/* Return the second highest RT task, NULL otherwise */ -static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu) +/* + * Return the highest pushable rq's task, which is suitable to be executed + * on the cpu, NULL otherwise + */ +static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu) { - struct task_struct *next = NULL; - struct sched_rt_entity *rt_se; - struct rt_prio_array *array; - struct rt_rq *rt_rq; - int idx; - - for_each_leaf_rt_rq(rt_rq, rq) { - array = &rt_rq->active; - idx = sched_find_first_bit(array->bitmap); -next_idx: - if (idx >= MAX_RT_PRIO) - continue; - if (next && next->prio <= idx) - continue; - list_for_each_entry(rt_se, array->queue + idx, run_list) { - struct task_struct *p; + struct plist_head *head = &rq->rt.pushable_tasks; + struct task_struct *p; - if (!rt_entity_is_task(rt_se)) - continue; + if (!has_pushable_tasks(rq)) + return NULL; - p = rt_task_of(rt_se); - if (pick_rt_task(rq, p, cpu)) { - next = p; - break; - } - } - if (!next) { - idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx+1); - goto next_idx; - } + plist_for_each_entry(p, head, pushable_tasks) { + if (pick_rt_task(rq, p, cpu)) + return p; } - return next; + return NULL; } static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask); -#ifdef CONFIG_SCHED_HMP -static int find_lowest_rq_hmp(struct task_struct *task) -{ - struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask); - int cpu_cost, min_cost = INT_MAX; - int best_cpu = -1; - int i; - - /* Make sure the mask is initialized first */ - if (unlikely(!lowest_mask)) - return best_cpu; - - if (task->nr_cpus_allowed == 1) - return best_cpu; /* No other targets possible */ - - if (!cpupri_find(&task_rq(task)->rd->cpupri, task, lowest_mask)) - return best_cpu; /* No targets found */ - - /* - * At this point we have built a mask of cpus representing the - * lowest priority tasks in the system. Now we want to elect - * the best one based on our affinity and topology. - */ - - /* Skip performance considerations and optimize for power. - * Worst case we'll be iterating over all CPUs here. CPU - * online mask should be taken care of when constructing - * the lowest_mask. - */ - for_each_cpu(i, lowest_mask) { - struct rq *rq = cpu_rq(i); - cpu_cost = power_cost_at_freq(i, ACCESS_ONCE(rq->min_freq)); - trace_sched_cpu_load(rq, idle_cpu(i), mostly_idle_cpu(i), - sched_irqload(i), cpu_cost, cpu_temp(i)); - - if (sched_boost() && capacity(rq) != max_capacity) - continue; - - if (cpu_cost < min_cost && !sched_cpu_high_irqload(i)) { - min_cost = cpu_cost; - best_cpu = i; - } - } - return best_cpu; -} - -#else /* CONFIG_SCHED_HMP */ - -static int find_lowest_rq_hmp(struct task_struct *task) -{ - return -1; -} - -#endif /* CONFIG_SCHED_HMP */ - static int find_lowest_rq(struct task_struct *task) { struct sched_domain *sd; - struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask); + struct cpumask *lowest_mask = this_cpu_ptr(local_cpu_mask); int this_cpu = smp_processor_id(); int cpu = task_cpu(task); - if (sched_enable_hmp) - return find_lowest_rq_hmp(task); - /* Make sure the mask is initialized first */ if (unlikely(!lowest_mask)) return -1; @@ -1710,7 +1640,7 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq) !cpumask_test_cpu(lowest_rq->cpu, tsk_cpus_allowed(task)) || task_running(rq, task) || - !task->on_rq)) { + !task_on_rq_queued(task))) { double_unlock_balance(rq, lowest_rq); lowest_rq = NULL; @@ -1744,7 +1674,7 @@ static struct task_struct *pick_next_pushable_task(struct rq *rq) BUG_ON(task_current(rq, p)); BUG_ON(p->nr_cpus_allowed <= 1); - BUG_ON(!p->on_rq); + BUG_ON(!task_on_rq_queued(p)); BUG_ON(!rt_task(p)); return p; @@ -1780,7 +1710,7 @@ static int push_rt_task(struct rq *rq) * just reschedule current. */ if (unlikely(next_task->prio < rq->curr->prio)) { - resched_task(rq->curr); + resched_curr(rq); return 0; } @@ -1827,7 +1757,7 @@ static int push_rt_task(struct rq *rq) activate_task(lowest_rq, next_task, 0); ret = 1; - resched_task(lowest_rq->curr); + resched_curr(lowest_rq); double_unlock_balance(rq, lowest_rq); @@ -1853,6 +1783,12 @@ static int pull_rt_task(struct rq *this_rq) if (likely(!rt_overloaded(this_rq))) return 0; + /* + * Match the barrier from rt_set_overloaded; this guarantees that if we + * see overloaded we must also see the rto_mask bit. + */ + smp_rmb(); + for_each_cpu(cpu, this_rq->rd->rto_mask) { if (this_cpu == cpu) continue; @@ -1878,12 +1814,10 @@ static int pull_rt_task(struct rq *this_rq) double_lock_balance(this_rq, src_rq); /* - * Are there still pullable RT tasks? + * We can pull only a task, which is pushable + * on its rq, and no others. */ - if (src_rq->rt.rt_nr_running <= 1) - goto skip; - - p = pick_next_highest_task_rt(src_rq, this_cpu); + p = pick_highest_pushable_task(src_rq, this_cpu); /* * Do we have an RT task that preempts @@ -1891,7 +1825,7 @@ static int pull_rt_task(struct rq *this_rq) */ if (p && (p->prio < this_rq->rt.highest_prio.curr)) { WARN_ON(p == src_rq->curr); - WARN_ON(!p->on_rq); + WARN_ON(!task_on_rq_queued(p)); /* * There's a chance that p is higher in priority @@ -1923,13 +1857,6 @@ static int pull_rt_task(struct rq *this_rq) return ret; } -static void pre_schedule_rt(struct rq *rq, struct task_struct *prev) -{ - /* Try to pull RT tasks here if we lower this rq's prio */ - if (rq->rt.highest_prio.curr > prev->prio) - pull_rt_task(rq); -} - static void post_schedule_rt(struct rq *rq) { push_rt_tasks(rq); @@ -1945,7 +1872,7 @@ static void task_woken_rt(struct rq *rq, struct task_struct *p) !test_tsk_need_resched(rq->curr) && has_pushable_tasks(rq) && p->nr_cpus_allowed > 1 && - rt_task(rq->curr) && + (dl_task(rq->curr) || rt_task(rq->curr)) && (rq->curr->nr_cpus_allowed < 2 || rq->curr->prio <= p->prio)) push_rt_tasks(rq); @@ -1959,7 +1886,7 @@ static void set_cpus_allowed_rt(struct task_struct *p, BUG_ON(!rt_task(p)); - if (!p->on_rq) + if (!task_on_rq_queued(p)) return; weight = cpumask_weight(new_mask); @@ -2025,14 +1952,14 @@ static void switched_from_rt(struct rq *rq, struct task_struct *p) * we may need to handle the pulling of RT tasks * now. */ - if (!p->on_rq || rq->rt.rt_nr_running) + if (!task_on_rq_queued(p) || rq->rt.rt_nr_running) return; if (pull_rt_task(rq)) - resched_task(rq->curr); + resched_curr(rq); } -void init_sched_rt_class(void) +void __init init_sched_rt_class(void) { unsigned int i; @@ -2041,7 +1968,6 @@ void init_sched_rt_class(void) GFP_KERNEL, cpu_to_node(i)); } } - #endif /* CONFIG_SMP */ /* @@ -2060,15 +1986,15 @@ static void switched_to_rt(struct rq *rq, struct task_struct *p) * If that current running task is also an RT task * then see if we can move to another run queue. */ - if (p->on_rq && rq->curr != p) { + if (task_on_rq_queued(p) && rq->curr != p) { #ifdef CONFIG_SMP - if (rq->rt.overloaded && push_rt_task(rq) && + if (p->nr_cpus_allowed > 1 && rq->rt.overloaded && /* Don't resched if we changed runqueues */ - rq != task_rq(p)) + push_rt_task(rq) && rq != task_rq(p)) check_resched = 0; #endif /* CONFIG_SMP */ if (check_resched && p->prio < rq->curr->prio) - resched_task(rq->curr); + resched_curr(rq); } } @@ -2079,7 +2005,7 @@ static void switched_to_rt(struct rq *rq, struct task_struct *p) static void prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio) { - if (!p->on_rq) + if (!task_on_rq_queued(p)) return; if (rq->curr == p) { @@ -2097,11 +2023,11 @@ prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio) * Only reschedule if p is still on the same runqueue. */ if (p->prio > rq->rt.highest_prio.curr && rq->curr == p) - resched_task(p); + resched_curr(rq); #else /* For UP simply resched on drop of prio */ if (oldprio < p->prio) - resched_task(p); + resched_curr(rq); #endif /* CONFIG_SMP */ } else { /* @@ -2110,7 +2036,7 @@ prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio) * then reschedule. */ if (p->prio < rq->curr->prio) - resched_task(rq->curr); + resched_curr(rq); } } @@ -2157,13 +2083,13 @@ static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued) p->rt.time_slice = sched_rr_timeslice; /* - * Requeue to the end of queue if we (and all of our ancestors) are the - * only element on the queue + * Requeue to the end of queue if we (and all of our ancestors) are not + * the only element on the queue */ for_each_sched_rt_entity(rt_se) { if (rt_se->run_list.prev != rt_se->run_list.next) { requeue_task_rt(rq, p, 0); - set_tsk_need_resched(p); + resched_curr(rq); return; } } @@ -2173,7 +2099,7 @@ static void set_curr_task_rt(struct rq *rq) { struct task_struct *p = rq->curr; - p->se.exec_start = rq->clock_task; + p->se.exec_start = rq_clock_task(rq); /* The running task is never eligible for pushing */ dequeue_pushable_task(rq, p); @@ -2207,7 +2133,6 @@ const struct sched_class rt_sched_class = { .set_cpus_allowed = set_cpus_allowed_rt, .rq_online = rq_online_rt, .rq_offline = rq_offline_rt, - .pre_schedule = pre_schedule_rt, .post_schedule = post_schedule_rt, .task_woken = task_woken_rt, .switched_from = switched_from_rt, @@ -2220,10 +2145,8 @@ const struct sched_class rt_sched_class = { .prio_changed = prio_changed_rt, .switched_to = switched_to_rt, -#ifdef CONFIG_SCHED_HMP - .inc_hmp_sched_stats = inc_hmp_sched_stats_rt, - .dec_hmp_sched_stats = dec_hmp_sched_stats_rt, -#endif + + .update_curr = update_curr_rt, }; #ifdef CONFIG_SCHED_DEBUG diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index bde547591ed3e..b5ef65d3e4c4e 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2,33 +2,37 @@ #include #include #include +#include #include #include #include #include +#include #include "cpupri.h" +#include "cpudeadline.h" #include "cpuacct.h" +struct rq; +struct cpuidle_state; + +/* task_struct::on_rq states: */ +#define TASK_ON_RQ_QUEUED 1 +#define TASK_ON_RQ_MIGRATING 2 + extern __read_mostly int scheduler_running; -/* - * Convert user-nice values [ -20 ... 0 ... 19 ] - * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ], - * and back. - */ -#define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20) -#define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20) -#define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio) +extern unsigned long calc_load_update; +extern atomic_long_t calc_load_tasks; -/* - * 'User priority' is the nice value converted to something we - * can work with better when scaling various scheduler parameters, - * it's a [ 0 ... 39 ] range. - */ -#define USER_PRIO(p) ((p)-MAX_RT_PRIO) -#define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio) -#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO)) +extern void calc_global_load_tick(struct rq *this_rq); +extern long calc_load_fold_active(struct rq *this_rq); + +#ifdef CONFIG_SMP +extern void update_cpu_load_active(struct rq *this_rq); +#else +static inline void update_cpu_load_active(struct rq *this_rq) { } +#endif /* * Helpers for converting nanosecond timing to jiffy resolution @@ -63,6 +67,13 @@ extern __read_mostly int scheduler_running; #define NICE_0_LOAD SCHED_LOAD_SCALE #define NICE_0_SHIFT SCHED_LOAD_SHIFT +/* + * Single value that decides SCHED_DEADLINE internal math precision. + * 10 -> just above 1us + * 9 -> just above 0.5us + */ +#define DL_SCALE (10) + /* * These are the 'tuning knobs' of the scheduler: */ @@ -82,11 +93,35 @@ static inline int rt_policy(int policy) return policy == SCHED_FIFO || policy == SCHED_RR; } +static inline int dl_policy(int policy) +{ + return policy == SCHED_DEADLINE; +} + static inline int task_has_rt_policy(struct task_struct *p) { return rt_policy(p->policy); } +static inline int task_has_dl_policy(struct task_struct *p) +{ + return dl_policy(p->policy); +} + +static inline bool dl_time_before(u64 a, u64 b) +{ + return (s64)(a - b) < 0; +} + +/* + * Tells if entity @a should preempt entity @b. + */ +static inline bool +dl_entity_preempt(struct sched_dl_entity *a, struct sched_dl_entity *b) +{ + return dl_time_before(a->deadline, b->deadline); +} + /* * This is the priority-queue data structure of the RT scheduling class: */ @@ -103,6 +138,50 @@ struct rt_bandwidth { struct hrtimer rt_period_timer; }; +void __dl_clear_params(struct task_struct *p); + +/* + * To keep the bandwidth of -deadline tasks and groups under control + * we need some place where: + * - store the maximum -deadline bandwidth of the system (the group); + * - cache the fraction of that bandwidth that is currently allocated. + * + * This is all done in the data structure below. It is similar to the + * one used for RT-throttling (rt_bandwidth), with the main difference + * that, since here we are only interested in admission control, we + * do not decrease any runtime while the group "executes", neither we + * need a timer to replenish it. + * + * With respect to SMP, the bandwidth is given on a per-CPU basis, + * meaning that: + * - dl_bw (< 100%) is the bandwidth of the system (group) on each CPU; + * - dl_total_bw array contains, in the i-eth element, the currently + * allocated bandwidth on the i-eth CPU. + * Moreover, groups consume bandwidth on each CPU, while tasks only + * consume bandwidth on the CPU they're running on. + * Finally, dl_total_bw_cpu is used to cache the index of dl_total_bw + * that will be shown the next time the proc or cgroup controls will + * be red. It on its turn can be changed by writing on its own + * control. + */ +struct dl_bandwidth { + raw_spinlock_t dl_runtime_lock; + u64 dl_runtime; + u64 dl_period; +}; + +static inline int dl_bandwidth_enabled(void) +{ + return sysctl_sched_rt_runtime >= 0; +} + +extern struct dl_bw *dl_bw_of(int i); + +struct dl_bw { + raw_spinlock_t lock; + u64 bw, total_bw; +}; + extern struct mutex sched_domains_mutex; #ifdef CONFIG_CGROUP_SCHED @@ -113,9 +192,6 @@ struct cfs_rq; struct rt_rq; extern struct list_head task_groups; -#ifdef VENDOR_EDIT -void skip_cfs_throttle(int skip); -#endif struct cfs_bandwidth { #ifdef CONFIG_CFS_BANDWIDTH @@ -125,7 +201,7 @@ struct cfs_bandwidth { #ifdef VENDOR_EDIT u64 quota_per_task; #endif - s64 hierarchal_quota; + s64 hierarchical_quota; u64 runtime_expires; int idle, timer_active; @@ -146,11 +222,6 @@ struct cfs_bandwidth { struct task_group { struct cgroup_subsys_state css; - bool notify_on_migrate; -#ifdef CONFIG_SCHED_HMP - bool upmigrate_discouraged; -#endif - #ifdef CONFIG_FAIR_GROUP_SCHED /* schedulable entities of this group on each cpu */ struct sched_entity **se; @@ -158,10 +229,11 @@ struct task_group { struct cfs_rq **cfs_rq; unsigned long shares; - atomic_t load_weight; - atomic64_t load_avg; +#ifdef CONFIG_SMP + atomic_long_t load_avg; atomic_t runnable_avg; #endif +#endif #ifdef CONFIG_RT_GROUP_SCHED struct sched_rt_entity **rt_se; @@ -254,15 +326,6 @@ struct cfs_bandwidth { }; #endif /* CONFIG_CGROUP_SCHED */ -#ifdef CONFIG_SCHED_HMP - -struct hmp_sched_stats { - int nr_big_tasks, nr_small_tasks; - u64 cumulative_runnable_avg; -}; - -#endif - /* CFS-related fields in a runqueue */ struct cfs_rq { struct load_weight load; @@ -288,27 +351,31 @@ struct cfs_rq { #endif #ifdef CONFIG_SMP -/* - * Load-tracking only depends on SMP, FAIR_GROUP_SCHED dependency below may be - * removed when useful for applications beyond shares distribution (e.g. - * load-balance). - */ -#ifdef CONFIG_FAIR_GROUP_SCHED /* * CFS Load tracking * Under CFS, load is tracked on a per-entity basis and aggregated up. * This allows for the description of both thread and group usage (in * the FAIR_GROUP_SCHED case). + * runnable_load_avg is the sum of the load_avg_contrib of the + * sched_entities on the rq. + * blocked_load_avg is similar to runnable_load_avg except that its + * the blocked sched_entities on the rq. + * utilization_load_avg is the sum of the average running time of the + * sched_entities on the rq. + * utilization_blocked_avg is the utilization equivalent of + * blocked_load_avg, i.e. the sum of running contributions of blocked + * sched_entities associated with the rq. */ - u64 runnable_load_avg, blocked_load_avg; - atomic64_t decay_counter, removed_load; + unsigned long runnable_load_avg, blocked_load_avg; + unsigned long utilization_load_avg, utilization_blocked_avg; + atomic64_t decay_counter; u64 last_decay; -#endif /* CONFIG_FAIR_GROUP_SCHED */ -/* These always depend on CONFIG_FAIR_GROUP_SCHED */ + atomic_long_t removed_load, removed_utilization; + #ifdef CONFIG_FAIR_GROUP_SCHED + /* Required to track per-cpu representation of a task_group */ u32 tg_runnable_contrib; - u64 tg_load_contrib; -#endif /* CONFIG_FAIR_GROUP_SCHED */ + unsigned long tg_load_contrib; /* * h_load = weight * f(tg) @@ -317,6 +384,9 @@ struct cfs_rq { * this group. */ unsigned long h_load; + u64 last_h_load_update; + struct sched_entity *h_load_next; +#endif /* CONFIG_FAIR_GROUP_SCHED */ #endif /* CONFIG_SMP */ #ifdef CONFIG_FAIR_GROUP_SCHED @@ -335,11 +405,6 @@ struct cfs_rq { struct task_group *tg; /* group that "owns" this runqueue */ #ifdef CONFIG_CFS_BANDWIDTH - -#ifdef CONFIG_SCHED_HMP - struct hmp_sched_stats hmp_stats; -#endif - int runtime_enabled; u64 runtime_expires; s64 runtime_remaining; @@ -380,6 +445,8 @@ struct rt_rq { int overloaded; struct plist_head pushable_tasks; #endif + int rt_queued; + int rt_throttled; u64 rt_time; u64 rt_runtime; @@ -390,11 +457,45 @@ struct rt_rq { unsigned long rt_nr_boosted; struct rq *rq; - struct list_head leaf_rt_rq_list; struct task_group *tg; #endif }; +/* Deadline class' related fields in a runqueue */ +struct dl_rq { + /* runqueue is an rbtree, ordered by deadline */ + struct rb_root rb_root; + struct rb_node *rb_leftmost; + + unsigned long dl_nr_running; + +#ifdef CONFIG_SMP + /* + * Deadline values of the currently executing and the + * earliest ready task on this rq. Caching these facilitates + * the decision wether or not a ready but not running task + * should migrate somewhere else. + */ + struct { + u64 curr; + u64 next; + } earliest_dl; + + unsigned long dl_nr_migratory; + int overloaded; + + /* + * Tasks on this rq that can be pushed away. They are kept in + * an rb-tree, ordered by tasks' deadlines, with caching + * of the leftmost (earliest deadline) element. + */ + struct rb_root pushable_dl_tasks_root; + struct rb_node *pushable_dl_tasks_leftmost; +#else + struct dl_bw dl_bw; +#endif +}; + #ifdef CONFIG_SMP /* @@ -412,12 +513,30 @@ struct root_domain { cpumask_var_t span; cpumask_var_t online; + /* Indicate more than one runnable task for any CPU */ + bool overload; + + /* Indicate one or more cpus over-utilized (tipping point) */ + bool overutilized; + + /* + * The bit corresponding to a CPU gets set here if such CPU has more + * than one runnable -deadline task (as it is below for RT tasks). + */ + cpumask_var_t dlo_mask; + atomic_t dlo_count; + struct dl_bw dl_bw; + struct cpudl cpudl; + /* * The "RT overload" flag: it gets set if a CPU has more than * one runnable RT task. */ cpumask_var_t rto_mask; struct cpupri cpupri; + + /* Maximum cpu capacity in the system. */ + unsigned long max_cpu_capacity; }; extern struct root_domain def_root_domain; @@ -440,9 +559,14 @@ struct rq { * remote CPUs use both these fields when doing load calculation. */ unsigned int nr_running; +#ifdef CONFIG_NUMA_BALANCING + unsigned int nr_numa_running; + unsigned int nr_preferred_running; +#endif #define CPU_LOAD_IDX_MAX 5 unsigned long cpu_load[CPU_LOAD_IDX_MAX]; unsigned long last_load_update_tick; + unsigned int misfit_task; #ifdef CONFIG_NO_HZ_COMMON u64 nohz_stamp; unsigned long nohz_flags; @@ -459,18 +583,14 @@ struct rq { struct cfs_rq cfs; struct rt_rq rt; + struct dl_rq dl; #ifdef CONFIG_FAIR_GROUP_SCHED /* list of leaf cfs_rq on this cpu: */ struct list_head leaf_cfs_rq_list; -#ifdef CONFIG_SMP - unsigned long h_load_throttle; -#endif /* CONFIG_SMP */ -#endif /* CONFIG_FAIR_GROUP_SCHED */ -#ifdef CONFIG_RT_GROUP_SCHED - struct list_head leaf_rt_rq_list; -#endif + struct sched_avg avg; +#endif /* CONFIG_FAIR_GROUP_SCHED */ /* * This is part of a global counter where only the total sum @@ -493,14 +613,14 @@ struct rq { struct root_domain *rd; struct sched_domain *sd; - unsigned long cpu_power; + unsigned long cpu_capacity; + unsigned long cpu_capacity_orig; unsigned char idle_balance; /* For active balancing */ int post_schedule; int active_balance; int push_cpu; - struct task_struct *push_task; struct cpu_stop_work active_balance_work; /* cpu of this runqueue: */ int cpu; @@ -512,43 +632,9 @@ struct rq { u64 age_stamp; u64 idle_stamp; u64 avg_idle; - int cstate, wakeup_latency, wakeup_energy; -#endif - -#ifdef CONFIG_SCHED_HMP - /* - * max_freq = user or thermal defined maximum - * max_possible_freq = maximum supported by hardware - */ - unsigned int cur_freq, max_freq, min_freq, max_possible_freq; - struct cpumask freq_domain_cpumask; - - struct hmp_sched_stats hmp_stats; - - int efficiency; /* Differentiate cpus with different IPC capability */ - int load_scale_factor; - int capacity; - int max_possible_capacity; - u64 window_start; - int prefer_idle; - u32 mostly_idle_load; - int mostly_idle_nr_run; - int mostly_idle_freq; - unsigned long hmp_flags; - - u64 cur_irqload; - u64 avg_irqload; - u64 irqload_ts; - -#ifdef CONFIG_SCHED_FREQ_INPUT - unsigned int old_busy_time; - int notifier_sent; -#endif -#endif -#ifdef CONFIG_SCHED_FREQ_INPUT - u64 curr_runnable_sum; - u64 prev_runnable_sum; + /* This is used to determine avg_idle's max value */ + u64 max_idle_balance_cost; #endif #ifdef CONFIG_IRQ_TIME_ACCOUNTING @@ -595,7 +681,11 @@ struct rq { struct llist_head wake_list; #endif - struct sched_avg avg; +#ifdef CONFIG_CPU_IDLE + /* Must be inspected within a rcu lock section */ + struct cpuidle_state *idle_state; + int idle_state_idx; +#endif }; static inline int cpu_of(struct rq *rq) @@ -607,16 +697,41 @@ static inline int cpu_of(struct rq *rq) #endif } -DECLARE_PER_CPU(struct rq, runqueues); +DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); #define cpu_rq(cpu) (&per_cpu(runqueues, (cpu))) -#define this_rq() (&__get_cpu_var(runqueues)) +#define this_rq() this_cpu_ptr(&runqueues) #define task_rq(p) cpu_rq(task_cpu(p)) #define cpu_curr(cpu) (cpu_rq(cpu)->curr) -#define raw_rq() (&__raw_get_cpu_var(runqueues)) +#define raw_rq() raw_cpu_ptr(&runqueues) + +static inline u64 __rq_clock_broken(struct rq *rq) +{ + return ACCESS_ONCE(rq->clock); +} + +static inline u64 rq_clock(struct rq *rq) +{ + lockdep_assert_held(&rq->lock); + return rq->clock; +} + +static inline u64 rq_clock_task(struct rq *rq) +{ + lockdep_assert_held(&rq->lock); + return rq->clock_task; +} + +#ifdef CONFIG_NUMA_BALANCING +extern void sched_setnuma(struct task_struct *p, int node); +extern int migrate_task_to(struct task_struct *p, int cpu); +extern int migrate_swap(struct task_struct *, struct task_struct *); +#endif /* CONFIG_NUMA_BALANCING */ #ifdef CONFIG_SMP +extern void sched_ttwu_pending(void); + #define rcu_dereference_check_sched_domain(p) \ rcu_dereference_check((p), \ lockdep_is_held(&sched_domains_mutex)) @@ -656,17 +771,36 @@ static inline struct sched_domain *highest_flag_domain(int cpu, int flag) return hsd; } +static inline struct sched_domain *lowest_flag_domain(int cpu, int flag) +{ + struct sched_domain *sd; + + for_each_domain(cpu, sd) { + if (sd->flags & flag) + break; + } + + return sd; +} + DECLARE_PER_CPU(struct sched_domain *, sd_llc); +DECLARE_PER_CPU(int, sd_llc_size); DECLARE_PER_CPU(int, sd_llc_id); +DECLARE_PER_CPU(struct sched_domain *, sd_numa); +DECLARE_PER_CPU(struct sched_domain *, sd_busy); +DECLARE_PER_CPU(struct sched_domain *, sd_asym); +DECLARE_PER_CPU(struct sched_domain *, sd_ea); -struct sched_group_power { +struct sched_group_capacity { atomic_t ref; /* - * CPU power of this group, SCHED_LOAD_SCALE being max power for a - * single CPU. + * CPU capacity of this group, SCHED_LOAD_SCALE being max capacity + * for a single CPU. */ - unsigned int power, power_orig; + unsigned long capacity; + unsigned long max_capacity; /* Max per-cpu capacity in group */ unsigned long next_update; + int imbalance; /* XXX unrelated to capacity but shared group state */ /* * Number of busy cpus in this group. */ @@ -680,7 +814,8 @@ struct sched_group { atomic_t ref; unsigned int group_weight; - struct sched_group_power *sgp; + struct sched_group_capacity *sgc; + struct sched_group_energy *sge; /* * The CPUs this group covers. @@ -703,7 +838,7 @@ static inline struct cpumask *sched_group_cpus(struct sched_group *sg) */ static inline struct cpumask *sched_group_mask(struct sched_group *sg) { - return to_cpumask(sg->sgp->cpumask); + return to_cpumask(sg->sgc->cpumask); } /** @@ -718,277 +853,55 @@ static inline unsigned int group_first_cpu(struct sched_group *group) extern int group_balance_cpu(struct sched_group *sg); /* - * Returns the rq capacity of any rq in a group. This does not play - * well with groups where rq capacity can change independently. - */ -#define group_rq_capacity(group) capacity(cpu_rq(group_first_cpu(group))) - -#endif /* CONFIG_SMP */ - -#include "stats.h" -#include "auto_group.h" - -extern void init_new_task_load(struct task_struct *p); - -#ifdef CONFIG_SCHED_HMP - -#define WINDOW_STATS_RECENT 0 -#define WINDOW_STATS_MAX 1 -#define WINDOW_STATS_MAX_RECENT_AVG 2 -#define WINDOW_STATS_AVG 3 -#define WINDOW_STATS_INVALID_POLICY 4 - -extern struct mutex policy_mutex; -extern unsigned int sched_ravg_window; -extern unsigned int sched_use_pelt; -extern unsigned int sched_disable_window_stats; -extern unsigned int sched_enable_hmp; -extern unsigned int max_possible_freq; -extern unsigned int min_max_freq; -extern unsigned int pct_task_load(struct task_struct *p); -extern unsigned int max_possible_efficiency; -extern unsigned int min_possible_efficiency; -extern unsigned int max_capacity; -extern unsigned int min_capacity; -extern unsigned int max_load_scale_factor; -extern unsigned int max_possible_capacity; -extern cpumask_t mpc_mask; -extern unsigned long capacity_scale_cpu_efficiency(int cpu); -extern unsigned long capacity_scale_cpu_freq(int cpu); -extern unsigned int sched_mostly_idle_load; -extern unsigned int sched_small_task; -extern unsigned int sched_upmigrate; -extern unsigned int sched_downmigrate; -extern unsigned int sched_init_task_load_pelt; -extern unsigned int sched_init_task_load_windows; -extern unsigned int sched_heavy_task; - -extern void reset_cpu_hmp_stats(int cpu, int reset_cra); -extern void fixup_nr_big_small_task(int cpu, int reset_stats); -unsigned int max_task_load(void); -extern void sched_account_irqtime(int cpu, struct task_struct *curr, - u64 delta, u64 wallclock); -unsigned int cpu_temp(int cpu); -extern unsigned int nr_eligible_big_tasks(int cpu); - -/* - * 'load' is in reference to "best cpu" at its best frequency. - * Scale that in reference to a given cpu, accounting for how bad it is - * in reference to "best cpu". + * Check that the per-cpu provided sd energy data is consistent for all cpus + * within the mask. */ -static inline u64 scale_load_to_cpu(u64 task_load, int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - task_load *= (u64)rq->load_scale_factor; - task_load /= 1024; - - return task_load; -} - -static inline int capacity(struct rq *rq) -{ - return rq->capacity; -} - -static inline void -inc_cumulative_runnable_avg(struct hmp_sched_stats *stats, - struct task_struct *p) -{ - u32 task_load; - - if (!sched_enable_hmp || sched_disable_window_stats) - return; - - task_load = sched_use_pelt ? p->se.avg.runnable_avg_sum_scaled : - (sched_disable_window_stats ? 0 : p->ravg.demand); - - stats->cumulative_runnable_avg += task_load; -} - -static inline void -dec_cumulative_runnable_avg(struct hmp_sched_stats *stats, - struct task_struct *p) +static inline void check_sched_energy_data(int cpu, sched_domain_energy_f fn, + const struct cpumask *cpumask) { - u32 task_load; + struct cpumask mask; + int i; - if (!sched_enable_hmp || sched_disable_window_stats) - return; + cpumask_xor(&mask, cpumask, get_cpu_mask(cpu)); - task_load = sched_use_pelt ? p->se.avg.runnable_avg_sum_scaled : - (sched_disable_window_stats ? 0 : p->ravg.demand); + for_each_cpu(i, &mask) { + int y; - stats->cumulative_runnable_avg -= task_load; + BUG_ON(fn(i)->nr_idle_states != fn(cpu)->nr_idle_states); - BUG_ON((s64)stats->cumulative_runnable_avg < 0); -} - -#define pct_to_real(tunable) \ - (div64_u64((u64)tunable * (u64)max_task_load(), 100)) - -#define real_to_pct(tunable) \ - (div64_u64((u64)tunable * (u64)100, (u64)max_task_load())) - -#define SCHED_HIGH_IRQ_TIMEOUT 3 -static inline u64 sched_irqload(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - s64 delta; - - delta = get_jiffies_64() - rq->irqload_ts; - /* - * Current context can be preempted by irq and rq->irqload_ts can be - * updated by irq context so that delta can be negative. - * But this is okay and we can safely return as this means there - * was recent irq occurrence. - */ - - if (delta < SCHED_HIGH_IRQ_TIMEOUT) - return rq->avg_irqload; - else - return 0; -} - -static inline int sched_cpu_high_irqload(int cpu) -{ - return sched_irqload(cpu) >= sysctl_sched_cpu_high_irqload; -} - -#else /* CONFIG_SCHED_HMP */ - -struct hmp_sched_stats; - -static inline unsigned int nr_eligible_big_tasks(int cpu) -{ - return 0; -} - -static inline int pct_task_load(struct task_struct *p) { return 0; } - -static inline int capacity(struct rq *rq) -{ - return SCHED_LOAD_SCALE; -} - -static inline void inc_cumulative_runnable_avg(struct hmp_sched_stats *stats, - struct task_struct *p) -{ -} - -static inline void dec_cumulative_runnable_avg(struct hmp_sched_stats *stats, - struct task_struct *p) -{ -} - -static inline unsigned long capacity_scale_cpu_efficiency(int cpu) -{ - return SCHED_LOAD_SCALE; -} - -static inline unsigned long capacity_scale_cpu_freq(int cpu) -{ - return SCHED_LOAD_SCALE; -} - -static inline void sched_account_irqtime(int cpu, struct task_struct *curr, - u64 delta, u64 wallclock) -{ -} - -static inline int sched_cpu_high_irqload(int cpu) { return 0; } - -#endif /* CONFIG_SCHED_HMP */ - -#ifdef CONFIG_SCHED_FREQ_INPUT -extern void check_for_freq_change(struct rq *rq); - -/* Is frequency of two cpus synchronized with each other? */ -static inline int same_freq_domain(int src_cpu, int dst_cpu) -{ - struct rq *rq = cpu_rq(src_cpu); - - if (src_cpu == dst_cpu) - return 1; - - return cpumask_test_cpu(dst_cpu, &rq->freq_domain_cpumask); -} - -#else /* CONFIG_SCHED_FREQ_INPUT */ - -#define sched_migration_fixup 0 - -static inline void check_for_freq_change(struct rq *rq) { } - -static inline int same_freq_domain(int src_cpu, int dst_cpu) -{ - return 1; -} - -#endif /* CONFIG_SCHED_FREQ_INPUT */ - -#ifdef CONFIG_SCHED_HMP - -#define BOOST_KICK 0 -#define CPU_RESERVED 1 - -static inline int is_reserved(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - return test_bit(CPU_RESERVED, &rq->hmp_flags); -} - -static inline int mark_reserved(int cpu) -{ - struct rq *rq = cpu_rq(cpu); - - /* Name boost_flags as hmp_flags? */ - return test_and_set_bit(CPU_RESERVED, &rq->hmp_flags); -} + for (y = 0; y < (fn(i)->nr_idle_states); y++) { + BUG_ON(fn(i)->idle_states[y].power != + fn(cpu)->idle_states[y].power); + } -static inline void clear_reserved(int cpu) -{ - struct rq *rq = cpu_rq(cpu); + BUG_ON(fn(i)->nr_cap_states != fn(cpu)->nr_cap_states); - clear_bit(CPU_RESERVED, &rq->hmp_flags); + for (y = 0; y < (fn(i)->nr_cap_states); y++) { + BUG_ON(fn(i)->cap_states[y].cap != + fn(cpu)->cap_states[y].cap); + BUG_ON(fn(i)->cap_states[y].power != + fn(cpu)->cap_states[y].power); + } + } } -int mostly_idle_cpu(int cpu); -extern void check_for_migration(struct rq *rq, struct task_struct *p); -extern void pre_big_small_task_count_change(const struct cpumask *cpus); -extern void post_big_small_task_count_change(const struct cpumask *cpus); -extern void set_hmp_defaults(void); -extern unsigned int power_cost_at_freq(int cpu, unsigned int freq); -extern void reset_all_window_stats(u64 window_start, unsigned int window_size); -extern void boost_kick(int cpu); -extern int sched_boost(void); - -#else /* CONFIG_SCHED_HMP */ - -#define sched_enable_hmp 0 -#define sched_freq_legacy_mode 1 - -static inline void check_for_migration(struct rq *rq, struct task_struct *p) { } -static inline void pre_big_small_task_count_change(void) { } -static inline void post_big_small_task_count_change(void) { } -static inline void set_hmp_defaults(void) { } - -static inline void clear_reserved(int cpu) { } +#else -#define power_cost_at_freq(...) 0 +static inline void sched_ttwu_pending(void) { } -#define trace_sched_cpu_load(...) +#endif /* CONFIG_SMP */ -#endif /* CONFIG_SCHED_HMP */ +#include "stats.h" +#include "auto_group.h" #ifdef CONFIG_CGROUP_SCHED /* * Return the group to which this tasks belongs. * - * We cannot use task_subsys_state() and friends because the cgroup - * subsystem changes that value before the cgroup_subsys::attach() method - * is called, therefore we cannot pin it and might observe the wrong value. + * We cannot use task_css() and friends because the cgroup subsystem + * changes that value before the cgroup_subsys::attach() method is called, + * therefore we cannot pin it and might observe the wrong value. * * The same is true for autogroup's p->signal->autogroup->tg, the autogroup * core changes this before calling sched_move_task(). @@ -1001,11 +914,6 @@ static inline struct task_group *task_group(struct task_struct *p) return p->sched_task_group; } -static inline bool task_notify_on_migrate(struct task_struct *p) -{ - return task_group(p)->notify_on_migrate; -} - /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { @@ -1031,10 +939,7 @@ static inline struct task_group *task_group(struct task_struct *p) { return NULL; } -static inline bool task_notify_on_migrate(struct task_struct *p) -{ - return false; -} + #endif /* CONFIG_CGROUP_SCHED */ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) @@ -1048,6 +953,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) */ smp_wmb(); task_thread_info(p)->cpu = cpu; + p->wake_cpu = cpu; #endif } @@ -1074,20 +980,10 @@ enum { #undef SCHED_FEAT #if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL) -static __always_inline bool static_branch__true(struct static_key *key) -{ - return static_key_true(key); /* Not out of line branch. */ -} - -static __always_inline bool static_branch__false(struct static_key *key) -{ - return static_key_false(key); /* Out of line branch. */ -} - #define SCHED_FEAT(name, enabled) \ static __always_inline bool static_branch_##name(struct static_key *key) \ { \ - return static_branch__##enabled(key); \ + return static_key_##enabled(key); \ } #include "features.h" @@ -1125,8 +1021,6 @@ static inline u64 global_rt_runtime(void) return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC; } - - static inline int task_current(struct rq *rq, struct task_struct *p) { return rq->curr == p; @@ -1141,6 +1035,15 @@ static inline int task_running(struct rq *rq, struct task_struct *p) #endif } +static inline int task_on_rq_queued(struct task_struct *p) +{ + return p->on_rq == TASK_ON_RQ_QUEUED; +} + +static inline int task_on_rq_migrating(struct task_struct *p) +{ + return p->on_rq == TASK_ON_RQ_MIGRATING; +} #ifndef prepare_arch_switch # define prepare_arch_switch(next) do { } while (0) @@ -1152,7 +1055,6 @@ static inline int task_running(struct rq *rq, struct task_struct *p) # define finish_arch_post_lock_switch() do { } while (0) #endif -#ifndef __ARCH_WANT_UNLOCKED_CTXSW static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next) { #ifdef CONFIG_SMP @@ -1190,35 +1092,6 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) raw_spin_unlock_irq(&rq->lock); } -#else /* __ARCH_WANT_UNLOCKED_CTXSW */ -static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next) -{ -#ifdef CONFIG_SMP - /* - * We can optimise this out completely for !SMP, because the - * SMP rebalancing from interrupt is the only thing that cares - * here. - */ - next->on_cpu = 1; -#endif - raw_spin_unlock(&rq->lock); -} - -static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) -{ -#ifdef CONFIG_SMP - /* - * After ->on_cpu is cleared, the task can be moved to a different CPU. - * We must ensure this doesn't happen until the switch is completely - * finished. - */ - smp_wmb(); - prev->on_cpu = 0; -#endif - local_irq_enable(); -} -#endif /* __ARCH_WANT_UNLOCKED_CTXSW */ - /* * wake flags */ @@ -1226,24 +1099,6 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) #define WF_FORK 0x02 /* child wakeup after fork */ #define WF_MIGRATED 0x4 /* internal use, task got migrated */ -static inline void update_load_add(struct load_weight *lw, unsigned long inc) -{ - lw->weight += inc; - lw->inv_weight = 0; -} - -static inline void update_load_sub(struct load_weight *lw, unsigned long dec) -{ - lw->weight -= dec; - lw->inv_weight = 0; -} - -static inline void update_load_set(struct load_weight *lw, unsigned long w) -{ - lw->weight = w; - lw->inv_weight = 0; -} - /* * To aid in avoiding the subversion of "niceness" due to uneven distribution * of tasks with abnormal "nice" values across CPUs the contribution that @@ -1304,9 +1159,13 @@ static const u32 prio_to_wmult[40] = { #else #define ENQUEUE_WAKING 0 #endif +#define ENQUEUE_REPLENISH 8 +#define ENQUEUE_WAKEUP_NEW 16 #define DEQUEUE_SLEEP 1 +#define RETRY_TASK ((void *)-1UL) + struct sched_class { const struct sched_class *next; @@ -1317,14 +1176,22 @@ struct sched_class { void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int flags); - struct task_struct * (*pick_next_task) (struct rq *rq); + /* + * It is the responsibility of the pick_next_task() method that will + * return the next task to call put_prev_task() on the @prev task or + * something equivalent. + * + * May return RETRY_TASK when it finds a higher prio class has runnable + * tasks. + */ + struct task_struct * (*pick_next_task) (struct rq *rq, + struct task_struct *prev); void (*put_prev_task) (struct rq *rq, struct task_struct *p); #ifdef CONFIG_SMP - int (*select_task_rq)(struct task_struct *p, int sd_flag, int flags); + int (*select_task_rq)(struct task_struct *p, int task_cpu, int sd_flag, int flags); void (*migrate_task_rq)(struct task_struct *p, int next_cpu); - void (*pre_schedule) (struct rq *this_rq, struct task_struct *task); void (*post_schedule) (struct rq *this_rq); void (*task_waking) (struct task_struct *task); void (*task_woken) (struct rq *this_rq, struct task_struct *task); @@ -1339,7 +1206,13 @@ struct sched_class { void (*set_curr_task) (struct rq *rq); void (*task_tick) (struct rq *rq, struct task_struct *p, int queued); void (*task_fork) (struct task_struct *p); + void (*task_dead) (struct task_struct *p); + /* + * The switched_from() call is allowed to drop rq->lock, therefore we + * cannot assume the switched_from/switched_to pair is serliazed by + * rq->lock. They are however serialized by p->pi_lock. + */ void (*switched_from) (struct rq *this_rq, struct task_struct *task); void (*switched_to) (struct rq *this_rq, struct task_struct *task); void (*prio_changed) (struct rq *this_rq, struct task_struct *task, @@ -1348,20 +1221,24 @@ struct sched_class { unsigned int (*get_rr_interval) (struct rq *rq, struct task_struct *task); + void (*update_curr) (struct rq *rq); + #ifdef CONFIG_FAIR_GROUP_SCHED void (*task_move_group) (struct task_struct *p, int on_rq); #endif -#ifdef CONFIG_SCHED_HMP - void (*inc_hmp_sched_stats)(struct rq *rq, struct task_struct *p); - void (*dec_hmp_sched_stats)(struct rq *rq, struct task_struct *p); -#endif }; +static inline void put_prev_task(struct rq *rq, struct task_struct *prev) +{ + prev->sched_class->put_prev_task(rq, prev); +} + #define sched_class_highest (&stop_sched_class) #define for_each_class(class) \ for (class = sched_class_highest; class; class = class->next) extern const struct sched_class stop_sched_class; +extern const struct sched_class dl_sched_class; extern const struct sched_class rt_sched_class; extern const struct sched_class fair_sched_class; extern const struct sched_class idle_sched_class; @@ -1369,77 +1246,149 @@ extern const struct sched_class idle_sched_class; #ifdef CONFIG_SMP -extern void update_group_power(struct sched_domain *sd, int cpu); +extern void update_group_capacity(struct sched_domain *sd, int cpu); -extern void trigger_load_balance(struct rq *rq, int cpu); -extern void idle_balance(int this_cpu, struct rq *this_rq); +extern void trigger_load_balance(struct rq *rq); -/* - * Only depends on SMP, FAIR_GROUP_SCHED may be removed when runnable_avg - * becomes useful in lb - */ -#if defined(CONFIG_FAIR_GROUP_SCHED) extern void idle_enter_fair(struct rq *this_rq); extern void idle_exit_fair(struct rq *this_rq); + #else -static inline void idle_enter_fair(struct rq *this_rq) {} -static inline void idle_exit_fair(struct rq *this_rq) {} + +static inline void idle_enter_fair(struct rq *rq) { } +static inline void idle_exit_fair(struct rq *rq) { } + #endif -#else /* CONFIG_SMP */ +#ifdef CONFIG_CPU_IDLE +static inline void idle_set_state(struct rq *rq, + struct cpuidle_state *idle_state) +{ + rq->idle_state = idle_state; +} + +static inline struct cpuidle_state *idle_get_state(struct rq *rq) +{ + WARN_ON(!rcu_read_lock_held()); + return rq->idle_state; +} + +static inline void idle_set_state_idx(struct rq *rq, int idle_state_idx) +{ + rq->idle_state_idx = idle_state_idx; +} + +static inline int idle_get_state_idx(struct rq *rq) +{ + WARN_ON(!rcu_read_lock_held()); + return rq->idle_state_idx; +} +#else +static inline void idle_set_state(struct rq *rq, + struct cpuidle_state *idle_state) +{ +} -static inline void idle_balance(int cpu, struct rq *rq) +static inline struct cpuidle_state *idle_get_state(struct rq *rq) { + return NULL; } +static inline void idle_set_state_idx(struct rq *rq, int idle_state_idx) +{ +} + +static inline int idle_get_state_idx(struct rq *rq) +{ + return -1; +} #endif -#ifdef CONFIG_SYSRQ_SCHED_DEBUG -extern void sysrq_sched_debug_show(void); +#if 0 +#ifndef arch_scale_cpu_capacity +static __always_inline +unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu) +{ + if (sd && (sd->flags & SD_SHARE_CPUCAPACITY) && (sd->span_weight > 1)) + return sd->smt_gain / sd->span_weight; + + return SCHED_CAPACITY_SCALE; +} +#endif +#endif + +unsigned long capacity_orig_of(int cpu); + +extern struct static_key __sched_energy_freq; +static inline bool sched_energy_freq(void) +{ + return static_key_false(&__sched_energy_freq); +} + +#ifdef CONFIG_CPU_FREQ_GOV_SCHED +void cpufreq_sched_set_cap(int cpu, unsigned long util); +void cpufreq_sched_reset_cap(int cpu); +#else +static inline void cpufreq_sched_set_cap(int cpu, unsigned long util) +{ } +static inline void cpufreq_sched_reset_cap(int cpu) +{ } #endif + +extern void sysrq_sched_debug_show(void); extern void sched_init_granularity(void); extern void update_max_interval(void); + +extern void init_sched_dl_class(void); extern void init_sched_rt_class(void); extern void init_sched_fair_class(void); +extern void init_sched_dl_class(void); -extern void resched_task(struct task_struct *p); +extern void resched_curr(struct rq *rq); extern void resched_cpu(int cpu); extern struct rt_bandwidth def_rt_bandwidth; extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime); -extern void update_idle_cpu_load(struct rq *this_rq); +extern struct dl_bandwidth def_dl_bandwidth; +extern void init_dl_bandwidth(struct dl_bandwidth *dl_b, u64 period, u64 runtime); +extern void init_dl_task_timer(struct sched_dl_entity *dl_se); -#ifdef CONFIG_PARAVIRT -static inline u64 steal_ticks(u64 steal) -{ - if (unlikely(steal > NSEC_PER_SEC)) - return div_u64(steal, TICK_NSEC); +unsigned long to_ratio(u64 period, u64 runtime); - return __iter_div_u64_rem(steal, TICK_NSEC, &steal); -} -#endif +extern void init_task_runnable_average(struct task_struct *p); -static inline void inc_nr_running(struct rq *rq) +static inline void add_nr_running(struct rq *rq, unsigned count) { - sched_update_nr_prod(cpu_of(rq), 1, true); - rq->nr_running++; + unsigned prev_nr = rq->nr_running; + + rq->nr_running = prev_nr + count; + + if (prev_nr < 2 && rq->nr_running >= 2) { +#ifdef CONFIG_SMP + if (!rq->rd->overload) + rq->rd->overload = true; +#endif #ifdef CONFIG_NO_HZ_FULL - if (rq->nr_running == 2) { if (tick_nohz_full_cpu(rq->cpu)) { - /* Order rq->nr_running write against the IPI */ - smp_wmb(); - smp_send_reschedule(rq->cpu); + /* + * Tick is needed if more than one task runs on a CPU. + * Send the target an IPI to kick it out of nohz mode. + * + * We assume that IPI implies full memory barrier and the + * new value of rq->nr_running is visible on reception + * from the target. + */ + tick_nohz_full_kick_cpu(rq->cpu); } - } #endif + } } -static inline void dec_nr_running(struct rq *rq) +static inline void sub_nr_running(struct rq *rq, unsigned count) { - sched_update_nr_prod(cpu_of(rq), 1, false); - rq->nr_running--; + rq->nr_running -= count; } static inline void rq_last_tick_reset(struct rq *rq) @@ -1494,14 +1443,18 @@ static inline int hrtick_enabled(struct rq *rq) #ifdef CONFIG_SMP extern void sched_avg_update(struct rq *rq); +extern unsigned long arch_scale_freq_capacity(struct sched_domain *sd, int cpu); +extern unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu); static inline void sched_rt_avg_update(struct rq *rq, u64 rt_delta) { - rq->rt_avg += rt_delta; + rq->rt_avg += rt_delta * arch_scale_freq_capacity(NULL, cpu_of(rq)); sched_avg_update(rq); } + #else static inline void sched_rt_avg_update(struct rq *rq, u64 rt_delta) { } static inline void sched_avg_update(struct rq *rq) { } +static inline void gov_cfs_update_cpu(int cpu) {} #endif extern void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period); @@ -1582,6 +1535,33 @@ static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest) lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_); } +static inline void double_lock(spinlock_t *l1, spinlock_t *l2) +{ + if (l1 > l2) + swap(l1, l2); + + spin_lock(l1); + spin_lock_nested(l2, SINGLE_DEPTH_NESTING); +} + +static inline void double_lock_irq(spinlock_t *l1, spinlock_t *l2) +{ + if (l1 > l2) + swap(l1, l2); + + spin_lock_irq(l1); + spin_lock_nested(l2, SINGLE_DEPTH_NESTING); +} + +static inline void double_raw_lock(raw_spinlock_t *l1, raw_spinlock_t *l2) +{ + if (l1 > l2) + swap(l1, l2); + + raw_spin_lock(l1); + raw_spin_lock_nested(l2, SINGLE_DEPTH_NESTING); +} + /* * double_rq_lock - safely lock two runqueues * @@ -1666,6 +1646,7 @@ extern void print_rt_stats(struct seq_file *m, int cpu); extern void init_cfs_rq(struct cfs_rq *cfs_rq); extern void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq); +extern void init_dl_rq(struct dl_rq *dl_rq, struct rq *rq); extern void cfs_bandwidth_usage_inc(void); extern void cfs_bandwidth_usage_dec(void); @@ -1679,9 +1660,6 @@ enum rq_nohz_flag_bits { #define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags) #endif -#define NOHZ_KICK_ANY 0 -#define NOHZ_KICK_RESTRICT 1 - #ifdef CONFIG_IRQ_TIME_ACCOUNTING DECLARE_PER_CPU(u64, cpu_hardirq_time); @@ -1730,16 +1708,3 @@ static inline u64 irq_time_read(int cpu) } #endif /* CONFIG_64BIT */ #endif /* CONFIG_IRQ_TIME_ACCOUNTING */ - -static inline void account_reset_rq(struct rq *rq) -{ -#ifdef CONFIG_IRQ_TIME_ACCOUNTING - rq->prev_irq_time = 0; -#endif -#ifdef CONFIG_PARAVIRT - rq->prev_steal_time = 0; -#endif -#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING - rq->prev_steal_time_rq = 0; -#endif -} diff --git a/kernel/sched/stats.c b/kernel/sched/stats.c index da98af347e8b6..a476bea17fbc6 100644 --- a/kernel/sched/stats.c +++ b/kernel/sched/stats.c @@ -142,4 +142,4 @@ static int __init proc_schedstat_init(void) proc_create("schedstat", 0, NULL, &proc_schedstat_operations); return 0; } -module_init(proc_schedstat_init); +subsys_initcall(proc_schedstat_init); diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h index 2ef90a51ec5e3..4ab7043396569 100644 --- a/kernel/sched/stats.h +++ b/kernel/sched/stats.h @@ -59,9 +59,9 @@ static inline void sched_info_reset_dequeued(struct task_struct *t) * from dequeue_task() to account for possible rq->clock skew across cpus. The * delta taken on each cpu would annul the skew. */ -static inline void sched_info_dequeued(struct task_struct *t) +static inline void sched_info_dequeued(struct rq *rq, struct task_struct *t) { - unsigned long long now = task_rq(t)->clock, delta = 0; + unsigned long long now = rq_clock(rq), delta = 0; if (unlikely(sched_info_on())) if (t->sched_info.last_queued) @@ -69,7 +69,7 @@ static inline void sched_info_dequeued(struct task_struct *t) sched_info_reset_dequeued(t); t->sched_info.run_delay += delta; - rq_sched_info_dequeued(task_rq(t), delta); + rq_sched_info_dequeued(rq, delta); } /* @@ -77,9 +77,9 @@ static inline void sched_info_dequeued(struct task_struct *t) * long it was waiting to run. We also note when it began so that we * can keep stats on how long its timeslice is. */ -static void sched_info_arrive(struct task_struct *t) +static void sched_info_arrive(struct rq *rq, struct task_struct *t) { - unsigned long long now = task_rq(t)->clock, delta = 0; + unsigned long long now = rq_clock(rq), delta = 0; if (t->sched_info.last_queued) delta = now - t->sched_info.last_queued; @@ -88,7 +88,7 @@ static void sched_info_arrive(struct task_struct *t) t->sched_info.last_arrival = now; t->sched_info.pcount++; - rq_sched_info_arrive(task_rq(t), delta); + rq_sched_info_arrive(rq, delta); } /* @@ -96,29 +96,30 @@ static void sched_info_arrive(struct task_struct *t) * the timestamp if it is already not set. It's assumed that * sched_info_dequeued() will clear that stamp when appropriate. */ -static inline void sched_info_queued(struct task_struct *t) +static inline void sched_info_queued(struct rq *rq, struct task_struct *t) { if (unlikely(sched_info_on())) if (!t->sched_info.last_queued) - t->sched_info.last_queued = task_rq(t)->clock; + t->sched_info.last_queued = rq_clock(rq); } /* - * Called when a process ceases being the active-running process, either - * voluntarily or involuntarily. Now we can calculate how long we ran. + * Called when a process ceases being the active-running process involuntarily + * due, typically, to expiring its time slice (this may also be called when + * switching to the idle task). Now we can calculate how long we ran. * Also, if the process is still in the TASK_RUNNING state, call * sched_info_queued() to mark that it has now again started waiting on * the runqueue. */ -static inline void sched_info_depart(struct task_struct *t) +static inline void sched_info_depart(struct rq *rq, struct task_struct *t) { - unsigned long long delta = task_rq(t)->clock - + unsigned long long delta = rq_clock(rq) - t->sched_info.last_arrival; - rq_sched_info_depart(task_rq(t), delta); + rq_sched_info_depart(rq, delta); if (t->state == TASK_RUNNING) - sched_info_queued(t); + sched_info_queued(rq, t); } /* @@ -127,32 +128,34 @@ static inline void sched_info_depart(struct task_struct *t) * the idle task.) We are only called when prev != next. */ static inline void -__sched_info_switch(struct task_struct *prev, struct task_struct *next) +__sched_info_switch(struct rq *rq, + struct task_struct *prev, struct task_struct *next) { - struct rq *rq = task_rq(prev); - /* * prev now departs the cpu. It's not interesting to record * stats about how efficient we were at scheduling the idle * process, however. */ if (prev != rq->idle) - sched_info_depart(prev); + sched_info_depart(rq, prev); if (next != rq->idle) - sched_info_arrive(next); + sched_info_arrive(rq, next); } static inline void -sched_info_switch(struct task_struct *prev, struct task_struct *next) +sched_info_switch(struct rq *rq, + struct task_struct *prev, struct task_struct *next) { if (unlikely(sched_info_on())) - __sched_info_switch(prev, next); + __sched_info_switch(rq, prev, next); } #else -#define sched_info_queued(t) do { } while (0) +#define sched_info_queued(rq, t) do { } while (0) #define sched_info_reset_dequeued(t) do { } while (0) -#define sched_info_dequeued(t) do { } while (0) -#define sched_info_switch(t, next) do { } while (0) +#define sched_info_dequeued(rq, t) do { } while (0) +#define sched_info_depart(rq, t) do { } while (0) +#define sched_info_arrive(rq, next) do { } while (0) +#define sched_info_switch(rq, t, next) do { } while (0) #endif /* CONFIG_SCHEDSTATS || CONFIG_TASK_DELAY_ACCT */ /* @@ -161,6 +164,39 @@ sched_info_switch(struct task_struct *prev, struct task_struct *next) * on CONFIG_SCHEDSTATS. */ +/** + * cputimer_running - return true if cputimer is running + * + * @tsk: Pointer to target task. + */ +static inline bool cputimer_running(struct task_struct *tsk) + +{ + struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; + + if (!cputimer->running) + return false; + + /* + * After we flush the task's sum_exec_runtime to sig->sum_sched_runtime + * in __exit_signal(), we won't account to the signal struct further + * cputime consumed by that task, even though the task can still be + * ticking after __exit_signal(). + * + * In order to keep a consistent behaviour between thread group cputime + * and thread group cputimer accounting, lets also ignore the cputime + * elapsing after __exit_signal() in any thread group timer running. + * + * This makes sure that POSIX CPU clocks and timers are synchronized, so + * that a POSIX CPU timer won't expire while the corresponding POSIX CPU + * clock delta is behind the expiring timer value. + */ + if (unlikely(!tsk->sighand)) + return false; + + return true; +} + /** * account_group_user_time - Maintain utime for a thread group. * @@ -176,7 +212,7 @@ static inline void account_group_user_time(struct task_struct *tsk, { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; - if (!cputimer->running) + if (!cputimer_running(tsk)) return; raw_spin_lock(&cputimer->lock); @@ -199,7 +235,7 @@ static inline void account_group_system_time(struct task_struct *tsk, { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; - if (!cputimer->running) + if (!cputimer_running(tsk)) return; raw_spin_lock(&cputimer->lock); @@ -222,7 +258,7 @@ static inline void account_group_exec_runtime(struct task_struct *tsk, { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; - if (!cputimer->running) + if (!cputimer_running(tsk)) return; raw_spin_lock(&cputimer->lock); diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c index d25c5f3f07b74..79ffec45a6acd 100644 --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -11,66 +11,43 @@ #ifdef CONFIG_SMP static int -select_task_rq_stop(struct task_struct *p, int sd_flag, int flags) +select_task_rq_stop(struct task_struct *p, int cpu, int sd_flag, int flags) { return task_cpu(p); /* stop tasks as never migrate */ } #endif /* CONFIG_SMP */ -#ifdef CONFIG_SCHED_HMP - -static void -inc_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p) -{ - inc_cumulative_runnable_avg(&rq->hmp_stats, p); -} - -static void -dec_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p) -{ - dec_cumulative_runnable_avg(&rq->hmp_stats, p); -} - -#else /* CONFIG_SCHED_HMP */ - -static inline void -inc_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p) { } - -static inline void -dec_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p) { } - -#endif /* CONFIG_SCHED_HMP */ - static void check_preempt_curr_stop(struct rq *rq, struct task_struct *p, int flags) { /* we're never preempted */ } -static struct task_struct *pick_next_task_stop(struct rq *rq) +static struct task_struct * +pick_next_task_stop(struct rq *rq, struct task_struct *prev) { struct task_struct *stop = rq->stop; - if (stop && stop->on_rq) { - stop->se.exec_start = rq->clock_task; - return stop; - } + if (!stop || !task_on_rq_queued(stop)) + return NULL; + + put_prev_task(rq, prev); + + stop->se.exec_start = rq_clock_task(rq); - return NULL; + return stop; } static void enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags) { - inc_nr_running(rq); - inc_hmp_sched_stats_stop(rq, p); + add_nr_running(rq, 1); } static void dequeue_task_stop(struct rq *rq, struct task_struct *p, int flags) { - dec_nr_running(rq); - dec_hmp_sched_stats_stop(rq, p); + sub_nr_running(rq, 1); } static void yield_task_stop(struct rq *rq) @@ -83,7 +60,7 @@ static void put_prev_task_stop(struct rq *rq, struct task_struct *prev) struct task_struct *curr = rq->curr; u64 delta_exec; - delta_exec = rq->clock_task - curr->se.exec_start; + delta_exec = rq_clock_task(rq) - curr->se.exec_start; if (unlikely((s64)delta_exec < 0)) delta_exec = 0; @@ -93,7 +70,7 @@ static void put_prev_task_stop(struct rq *rq, struct task_struct *prev) curr->se.sum_exec_runtime += delta_exec; account_group_exec_runtime(curr, delta_exec); - curr->se.exec_start = rq->clock_task; + curr->se.exec_start = rq_clock_task(rq); cpuacct_charge(curr, delta_exec); } @@ -105,7 +82,7 @@ static void set_curr_task_stop(struct rq *rq) { struct task_struct *stop = rq->stop; - stop->se.exec_start = rq->clock_task; + stop->se.exec_start = rq_clock_task(rq); } static void switched_to_stop(struct rq *rq, struct task_struct *p) @@ -125,11 +102,15 @@ get_rr_interval_stop(struct rq *rq, struct task_struct *task) return 0; } +static void update_curr_stop(struct rq *rq) +{ +} + /* * Simple, special scheduling class for the per-CPU stop tasks: */ const struct sched_class stop_sched_class = { - .next = &rt_sched_class, + .next = &dl_sched_class, .enqueue_task = enqueue_task_stop, .dequeue_task = dequeue_task_stop, @@ -151,8 +132,5 @@ const struct sched_class stop_sched_class = { .prio_changed = prio_changed_stop, .switched_to = switched_to_stop, -#ifdef CONFIG_SCHED_HMP - .inc_hmp_sched_stats = inc_hmp_sched_stats_stop, - .dec_hmp_sched_stats = dec_hmp_sched_stats_stop, -#endif + .update_curr = update_curr_stop, }; diff --git a/kernel/sched/tune.c b/kernel/sched/tune.c new file mode 100644 index 0000000000000..79de07ed75759 --- /dev/null +++ b/kernel/sched/tune.c @@ -0,0 +1,711 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "sched.h" +#include "tune.h" + +unsigned int sysctl_sched_cfs_boost __read_mostly = 0; + +/* Performance Boost region (B) threshold params */ +static int perf_boost_idx; + +/* Performance Constraint region (C) threshold params */ +static int perf_constrain_idx; + +/** + * Performance-Energy (P-E) Space thresholds constants + */ +struct threshold_params { + int nrg_gain; + int cap_gain; +}; + +/* + * System specific P-E space thresholds constants + */ +static struct threshold_params +threshold_gains[] = { + { 0, 4 }, /* >= 0% */ + { 1, 4 }, /* >= 10% */ + { 2, 4 }, /* >= 20% */ + { 3, 4 }, /* >= 30% */ + { 4, 4 }, /* >= 40% */ + { 4, 3 }, /* >= 50% */ + { 4, 2 }, /* >= 60% */ + { 4, 1 }, /* >= 70% */ + { 4, 0 }, /* >= 80% */ + { 4, 0 } /* >= 90% */ +}; + +/* + * System energy normalization constants + */ +struct target_nrg { + unsigned long min_power; + unsigned long max_power; + unsigned long nrg_shift; + unsigned long nrg_mult; +}; + +/* + * Target specific system energy normalization constants + * NOTE: These values are specific for ARM TC2 and they are derived from the + * energy model data defined in: arch/arm/kernel/topology.c + */ +static struct target_nrg +schedtune_target_nrg = { + + /* + * TC2 Min CPUs power: + * all CPUs idle, all clusters in deep idle: + * 0 * 3 + 0 * 2 + 10 + 25 + */ + .min_power = 35, + + /* + * TC2 Max CPUs power: + * all CPUs fully utilized while running at max OPP: + * 1024 * 3 + 6997 * 2 + 4905 + 15200 + */ + + .max_power = 37171, + + /* + * Fast integer division by constant: + * Constant : Max - Min (C) = 37171 - 35 = 37136 + * Precision : 0.1% (P) = 0.1 + * Reference : C * 100 / P (R) = 3713600 + * + * Thus: + * Shift bifs : ceil(log(R,2)) (S) = 26 + * Mult const : round(2^S/C) (M) = 1807 + * + * This allows to compute the normalized energy: + * system_energy / C + * as: + * (system_energy * M) >> S + */ + .nrg_shift = 26, /* S */ + .nrg_mult = 1807, /* M */ +}; + +/* + * System energy normalization + * Returns the normalized value, in the range [0..SCHED_LOAD_SCALE], + * corresponding to the specified energy variation. + */ +int +schedtune_normalize_energy(int energy_diff) +{ + long long normalized_nrg = energy_diff; + int max_delta; + + /* Check for boundaries */ + max_delta = schedtune_target_nrg.max_power; + max_delta -= schedtune_target_nrg.min_power; + WARN_ON(abs(energy_diff) >= max_delta); + + /* Scale by energy magnitude */ + normalized_nrg <<= SCHED_LOAD_SHIFT; + + /* Normalize on max energy for target platform */ + normalized_nrg *= schedtune_target_nrg.nrg_mult; + normalized_nrg >>= schedtune_target_nrg.nrg_shift; + + return normalized_nrg; +} + +static int +__schedtune_accept_deltas(int nrg_delta, int cap_delta, + int perf_boost_idx, int perf_constrain_idx) { + int energy_payoff; + + /* Performance Boost (B) region */ + if (nrg_delta > 0 && cap_delta > 0) { + /* + * energy_payoff criteria: + * cap_delta / nrg_delta > cap_gain / nrg_gain + * which is: + * nrg_delta * cap_gain < cap_delta * nrg_gain + */ + energy_payoff = cap_delta * threshold_gains[perf_boost_idx].nrg_gain; + energy_payoff -= nrg_delta * threshold_gains[perf_boost_idx].cap_gain; + + trace_sched_tune_filter( + threshold_gains[perf_boost_idx].nrg_gain, + threshold_gains[perf_boost_idx].cap_gain, + energy_payoff, 8); + + return energy_payoff; + } + + /* Performance Constraint (C) region */ + if (nrg_delta < 0 && cap_delta < 0) { + /* + * energy_payoff criteria: + * cap_delta / nrg_delta < cap_gain / nrg_gain + * which is: + * nrg_delta * cap_gain > cap_delta * nrg_gain + */ + energy_payoff = nrg_delta * threshold_gains[perf_constrain_idx].cap_gain; + energy_payoff -= cap_delta * threshold_gains[perf_constrain_idx].nrg_gain; + + trace_sched_tune_filter( + threshold_gains[perf_constrain_idx].nrg_gain, + threshold_gains[perf_constrain_idx].cap_gain, + energy_payoff, 6); + + return energy_payoff; + } + + /* Default: reject schedule candidate */ + return -INT_MAX; +} + +#ifdef CONFIG_CGROUP_SCHEDTUNE + +/* + * EAS scheduler tunables for task groups. + */ + +/* SchdTune tunables for a group of tasks */ +struct schedtune { + /* SchedTune CGroup subsystem */ + struct cgroup_subsys_state css; + + /* Boost group allocated ID */ + int idx; + + /* Boost value for tasks on that SchedTune CGroup */ + int boost; + + /* Performance Boost (B) region threshold params */ + int perf_boost_idx; + + /* Performance Constraint (C) region threshold params */ + int perf_constrain_idx; +}; + +static inline struct schedtune *css_st(struct cgroup_subsys_state *css) +{ + return css ? container_of(css, struct schedtune, css) : NULL; +} + +static inline struct schedtune *cgroup_st(struct cgroup *cgrp) { + return container_of(cgroup_subsys_state(cgrp, schedtune_subsys_id), struct schedtune, css); +} + +static inline struct schedtune *task_schedtune(struct task_struct *tsk) +{ + return cgroup_st(task_cgroup(tsk, schedtune_subsys_id)); +} + +#if 0 +static inline struct schedtune *parent_st(struct schedtune *st) +{ + return css_st(st->css.parent); +} +#endif + +/* + * SchedTune root control group + * The root control group is used to defined a system-wide boosting tuning, + * which is applied to all tasks in the system. + * Task specific boosting tuning could be specified by creating and + * configuring a child control group under the root one. + * By default, system-wide boosting is disabled, i.e. no boosting is applied + * to all tasks not into a child control group. + */ +static struct schedtune +root_schedtune = { + .boost = 0, + .perf_boost_idx = 0, + .perf_constrain_idx = 0, +}; + +int +schedtune_accept_deltas(int nrg_delta, int cap_delta, struct task_struct *task) { + struct schedtune *ct; + int perf_boost_idx; + int perf_constrain_idx; + + /* Optimal (O) region */ + if (nrg_delta < 0 && cap_delta > 0) { + trace_sched_tune_filter(0, 0, 1, 0); + return INT_MAX; + } + + /* Suboptimal (S) region */ + if (nrg_delta > 0 && cap_delta < 0) { + trace_sched_tune_filter(0, 0, -1, 5); + return -INT_MAX; + } + + /* Get task specific perf Boost/Constraints indexes */ + rcu_read_lock(); + ct = task_schedtune(task); + perf_boost_idx = ct->perf_boost_idx; + perf_constrain_idx = ct->perf_constrain_idx; + rcu_read_unlock(); + + return __schedtune_accept_deltas(nrg_delta, cap_delta, + perf_boost_idx, perf_constrain_idx); + +} + +/* + * Maximum number of boost groups to support + * When per-task boosting is used we still allows only limited number of + * boost groups for two main reasons: + * 1. on a real system we usually have only few classes of workloads which + * make sense to boost with different values (e.g. backgroud vs foreground + * tasks, interactive vs low-priority tasks) + * 2. a limited number allows for a simpler and more memory/time efficient + * implementation especially for the computation of the per-CPU boost + * value + */ +#define BOOSTGROUPS_COUNT 16 + +/* Array of configured boostgroups */ +static struct schedtune *allocated_group[BOOSTGROUPS_COUNT] = { + &root_schedtune, + NULL, +}; + +/* SchedTune boost groups + * Each CPU in the system could be affected by multiple boost groups, for + * example when a CPU has two RUNNABLE tasks beloging to two different boost + * groups and thus likely with different boost values. + * This data structure keep track of all the boost groups which could impact + * on a CPU. + * Since on each system we expect only a limited number of boost + * groups, here we use a simple array to keep track of the metrics required to + * compute the maximum per-CPU boosting value. + */ +struct boost_groups { + /* Maximum boost value for all RUNNABLE tasks on a CPU */ + bool idle; + unsigned boost_max; + struct { + /* The boost for tasks on that boost group */ + unsigned boost; + /* Count of RUNNABLE tasks on that boost group */ + unsigned tasks; + } group[BOOSTGROUPS_COUNT]; +}; + +/* Boost groups affecting each CPU in the system */ +DEFINE_PER_CPU(struct boost_groups, cpu_boost_groups); + +static void +schedtune_cpu_update(int cpu) +{ + struct boost_groups *bg; + unsigned boost_max; + int idx; + + bg = &per_cpu(cpu_boost_groups, cpu); + + /* The root boost group is always active */ + boost_max = bg->group[0].boost; + for (idx = 1; idx < BOOSTGROUPS_COUNT; ++idx) { + /* + * A boost group affects a CPU only if it has + * RUNNABLE tasks on that CPU + */ + if (bg->group[idx].tasks == 0) + continue; + + boost_max = max(boost_max, bg->group[idx].boost); + } + + bg->boost_max = boost_max; +} + +static int +schedtune_boostgroup_update(int idx, int boost) +{ + struct boost_groups *bg; + int cur_boost_max; + int old_boost; + int cpu; + + /* Update per CPU boost groups */ + for_each_possible_cpu(cpu) { + bg = &per_cpu(cpu_boost_groups, cpu); + + /* + * Keep track of current boost values to compute the per CPU + * maximum only when it has been affected by the new value of + * the updated boost group + */ + cur_boost_max = bg->boost_max; + old_boost = bg->group[idx].boost; + + /* Update the boost value of this boost group */ + bg->group[idx].boost = boost; + + /* Check if this update increase current max */ + if (boost > cur_boost_max && bg->group[idx].tasks) { + bg->boost_max = boost; + trace_sched_tune_boostgroup_update(cpu, 1, bg->boost_max); + continue; + } + + /* Check if this update has decreased current max */ + if (cur_boost_max == old_boost && old_boost > boost) { + schedtune_cpu_update(cpu); + trace_sched_tune_boostgroup_update(cpu, -1, bg->boost_max); + continue; + } + + trace_sched_tune_boostgroup_update(cpu, 0, bg->boost_max); + } + + return 0; +} + +void +schedtune_idle(int cpu) +{ + int idx; + struct boost_groups *bg; + + bg = &per_cpu(cpu_boost_groups, cpu); + if (!bg->idle) { + bg->idle = true; + for (idx = 1; idx < BOOSTGROUPS_COUNT; ++idx) { + bg->group[idx].tasks = 0; + } + } +} + +static inline void +schedtune_tasks_update(struct task_struct *p, int cpu, int idx, int task_count) +{ + struct boost_groups *bg; + int tasks; + + bg = &per_cpu(cpu_boost_groups, cpu); + bg->idle = false; + + /* Update boosted tasks count while avoiding to make it negative */ + if (task_count < 0 && bg->group[idx].tasks <= -task_count) + bg->group[idx].tasks = 0; + else + bg->group[idx].tasks += task_count; + + /* Boost group activation or deactivation on that RQ */ + tasks = bg->group[idx].tasks; + if (tasks == 1 || tasks == 0) + schedtune_cpu_update(cpu); + + trace_sched_tune_tasks_update(p, cpu, tasks, idx, + bg->group[idx].boost, bg->boost_max); +} + +/* + * NOTE: This function must be called while holding the lock on the CPU RQ + */ +void schedtune_enqueue_task(struct task_struct *p, int cpu) +{ + struct schedtune *st; + int idx; + + /* + * When a task is marked PF_EXITING by do_exit() it's going to be + * dequeued and enqueued multiple times in the exit path. + * Thus we avoid any further update, since we do not want to change + * CPU boosting while the task is exiting. + */ + if (p->flags & PF_EXITING) + return; + + /* Get task boost group */ + rcu_read_lock(); + st = task_schedtune(p); + idx = st->idx; + rcu_read_unlock(); + + schedtune_tasks_update(p, cpu, idx, 1); +} + +/* + * NOTE: This function must be called while holding the lock on the CPU RQ + */ +void schedtune_dequeue_task(struct task_struct *p, int cpu) +{ + struct schedtune *st; + int idx; + + /* + * When a task is marked PF_EXITING by do_exit() it's going to be + * dequeued and enqueued multiple times in the exit path. + * Thus we avoid any further update, since we do not want to change + * CPU boosting while the task is exiting. + * The last dequeue will be done by cgroup exit() callback. + */ + if (p->flags & PF_EXITING) + return; + + /* Get task boost group */ + rcu_read_lock(); + st = task_schedtune(p); + idx = st->idx; + rcu_read_unlock(); + + schedtune_tasks_update(p, cpu, idx, -1); +} + +int schedtune_taskgroup_boost(struct task_struct *p) +{ + struct schedtune *ct; + int task_boost; + + rcu_read_lock(); + ct = task_schedtune(p); + task_boost = ct->boost; + rcu_read_unlock(); + + return task_boost; +} + +int schedtune_cpu_boost(int cpu) +{ + struct boost_groups *bg; + bg = &per_cpu(cpu_boost_groups, cpu); + return bg->boost_max; +} + +static u64 +boost_read(struct cgroup *cgrp, struct cftype *cft) +{ + struct schedtune *st = cgroup_st(cgrp); + return st->boost; +} + +static int +boost_write(struct cgroup *cgrp, struct cftype *cft, + u64 boost) +{ + struct schedtune *st = cgroup_st(cgrp); + int err = 0; + + if (boost < 0 || boost > 100) { + err = -EINVAL; + goto out; + } + + st->boost = boost; + if (cgrp == (void *)&root_schedtune.css) + sysctl_sched_cfs_boost = boost; + + if (boost == 100) + boost = 99; + + /* Performance Boost (B) region threshold params */ + st->perf_boost_idx = boost; + st->perf_boost_idx /= 10; + + /* Performance Constraint (C) region threshold params */ + st->perf_constrain_idx = 100 - boost; + st->perf_constrain_idx /= 10; + + /* Update CPU boost */ + schedtune_boostgroup_update(st->idx, st->boost); + trace_sched_tune_config(st->boost, + threshold_gains[st->perf_boost_idx].nrg_gain, + threshold_gains[st->perf_boost_idx].cap_gain, + threshold_gains[st->perf_constrain_idx].nrg_gain, + threshold_gains[st->perf_constrain_idx].cap_gain); +out: + return err; +} + +static struct cftype files[] = { + { + .name = "boost", + .read_u64 = boost_read, + .write_u64 = boost_write, + }, + { } /* terminate */ +}; + +static int +schedtune_boostgroup_init(struct schedtune *st) +{ + struct boost_groups *bg; + int cpu; + + /* Keep track of allocated boost group */ + allocated_group[st->idx] = st; + + /* Initialize the per CPU boost groups */ + for_each_possible_cpu(cpu) { + bg = &per_cpu(cpu_boost_groups, cpu); + bg->group[st->idx].boost = 0; + bg->group[st->idx].tasks = 0; + } + + return 0; +} + +static int +schedtune_init(void) +{ + struct boost_groups *bg; + int cpu; + + /* Initialize the per CPU boost groups */ + for_each_possible_cpu(cpu) { + bg = &per_cpu(cpu_boost_groups, cpu); + memset(bg, 0 , sizeof(struct boost_groups)); + } + + pr_info(" schedtune configured to support %d boost groups\n", + BOOSTGROUPS_COUNT); + return 0; +} + +static struct cgroup_subsys_state * +schedtune_css_alloc(struct cgroup *cgrp) +{ + struct schedtune *st; + int idx; + + if (!cgrp->parent) { + schedtune_init(); + return &root_schedtune.css; + } + + /* Allows only single level hierachies */ +#if 0 + if (cgrp->parent != (void *)&root_schedtune.css) { + pr_err("Nested SchedTune boosting groups not allowed\n"); + return ERR_PTR(-ENOMEM); + } +#endif + /* Allows only a limited number of boosting groups */ + for (idx = 1; idx < BOOSTGROUPS_COUNT; ++idx) + if (allocated_group[idx] == NULL) + break; + if (idx == BOOSTGROUPS_COUNT) { + pr_err("Trying to create more than %d SchedTune boosting groups\n", + BOOSTGROUPS_COUNT); + return ERR_PTR(-ENOSPC); + } + + st = kzalloc(sizeof(*st), GFP_KERNEL); + if (!st) + goto out; + + /* Initialize per CPUs boost group support */ + st->idx = idx; + if (schedtune_boostgroup_init(st)) + goto release; + + return &st->css; + +release: + kfree(st); +out: + return ERR_PTR(-ENOMEM); +} + +static void +schedtune_boostgroup_release(struct schedtune *st) +{ + /* Reset this group boost */ + schedtune_boostgroup_update(st->idx, 0); + + /* Keep track of allocated boost group */ + allocated_group[st->idx] = NULL; +} + +static void +schedtune_css_free(struct cgroup *cgrp) +{ + struct schedtune *st = cgroup_st(cgrp); + schedtune_boostgroup_release(st); + kfree(st); +} + +static void +schedtune_exit(struct cgroup *cgrp, + struct cgroup *old_css, + struct task_struct *tsk) +{ + struct schedtune *old_st = cgroup_st(old_css); + int cpu = task_cpu(tsk); + + schedtune_tasks_update(tsk, cpu, old_st->idx, -1); +} + +static int +schedtune_allow_attach(struct cgroup *cgrp, struct cgroup_taskset *tset) { + // TJK: too permissive, but ok for now + return 0; +} + +struct cgroup_subsys schedtune_subsys = { + .name = "schedtune", + .css_alloc = schedtune_css_alloc, + .css_free = schedtune_css_free, + .exit = schedtune_exit, + .allow_attach = schedtune_allow_attach, + .base_cftypes = files, + .subsys_id = schedtune_subsys_id, + .early_init = 1, +}; + +#else /* CONFIG_CGROUP_SCHEDTUNE */ + +int +schedtune_accept_deltas(int nrg_delta, int cap_delta) { + + /* Optimal (O) region */ + if (nrg_delta < 0 && cap_delta > 0) { + trace_printk("schedtune_filter: region=O ngain=0 pgain=0 nrg_payoff=-1"); + return INT_MAX; + } + + /* Suboptimal (S) region */ + if (nrg_delta > 0 && cap_delta < 0) { + trace_printk("schedtune_filter: region=S ngain=0 pgain=0 nrg_payoff=1"); + return -INT_MAX; + } + + return __schedtune_accept_deltas(nrg_delta, cap_delta, + perf_boost_idx, perf_constrain_idx); + +} + +#endif /* CONFIG_CGROUP_SCHEDTUNE */ + +int +sysctl_sched_cfs_boost_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret || !write) + return ret; + + /* Performance Boost (B) region threshold params */ + perf_boost_idx = sysctl_sched_cfs_boost; + perf_boost_idx /= 10; + + /* Performance Constraint (C) region threshold params */ + perf_constrain_idx = 100 - sysctl_sched_cfs_boost; + perf_constrain_idx /= 10; + + return 0; +} + diff --git a/kernel/sched/tune.h b/kernel/sched/tune.h new file mode 100644 index 0000000000000..2332d30cf383a --- /dev/null +++ b/kernel/sched/tune.h @@ -0,0 +1,35 @@ + +#ifdef CONFIG_SCHED_TUNE + +extern int schedtune_normalize_energy(int energy); + +#ifdef CONFIG_CGROUP_SCHEDTUNE + +extern int schedtune_taskgroup_boost(struct task_struct *tsk); +extern int schedtune_cpu_boost(int cpu); +extern void schedtune_idle(int cpu); + +extern void schedtune_enqueue_task(struct task_struct *p, int cpu); +extern void schedtune_dequeue_task(struct task_struct *p, int cpu); + +extern int schedtune_accept_deltas(int nrg_delta, int cap_delta, + struct task_struct *task); + +#else /* CONFIG_CGROUP_SCHEDTUNE */ + +extern int schedtune_accept_deltas(int nrg_delta, int cap_delta); + +#define schedtune_enqueue_task(task, cpu) while(0){} +#define schedtune_dequeue_task(task, cpu) while(0){} + +#endif /* CONFIG_CGROUP_SCHEDTUNE */ + +#else /* CONFIG_SCHED_TUNE */ + +#define schedtune_normalize_energy(energy) energy +#define schedtune_accept_deltas(nrg_delta, cap_delta) nrg_delta + +#define schedtune_enqueue_task(task, cpu) while(0){} +#define schedtune_dequeue_task(task, cpu) while(0){} + +#endif /* CONFIG_SCHED_TUNE */ diff --git a/kernel/wait.c b/kernel/sched/wait.c similarity index 100% rename from kernel/wait.c rename to kernel/sched/wait.c diff --git a/kernel/smp.c b/kernel/smp.c index dec409c83ddba..7a2c5e3ebe9f8 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include "smpboot.h" @@ -76,7 +78,7 @@ hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu) return NOTIFY_OK; } -static struct notifier_block __cpuinitdata hotplug_cfd_notifier = { +static struct notifier_block hotplug_cfd_notifier = { .notifier_call = hotplug_cfd, }; @@ -273,6 +275,18 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info, } EXPORT_SYMBOL(smp_call_function_single); +int smp_call_function_single_async(int cpu, struct call_single_data *csd) +{ + int err = 0; + + preempt_disable(); + generic_exec_single(cpu, csd, 0); + preempt_enable(); + + return err; +} +EXPORT_SYMBOL_GPL(smp_call_function_single_async); + /* * smp_call_function_any - Run a function on any of the given cpus * @mask: The mask of cpus it can run on. @@ -719,3 +733,88 @@ void kick_all_cpus_sync(void) smp_call_function(do_nothing, NULL, 1); } EXPORT_SYMBOL_GPL(kick_all_cpus_sync); + + + +#ifdef CONFIG_CPU_FREQ + +static DEFINE_PER_CPU(atomic_long_t, cpu_max_freq); +DEFINE_PER_CPU(atomic_long_t, cpu_freq_capacity); + +/* + * Scheduler load-tracking scale-invariance + * + * Provides the scheduler with a scale-invariance correction factor that + * compensates for frequency scaling through arch_scale_freq_capacity() + * (implemented in topology.c). + */ +static inline +void scale_freq_capacity(int cpu, unsigned long curr, unsigned long max) +{ + unsigned long capacity; + + if (!max) + return; + + capacity = (curr << SCHED_CAPACITY_SHIFT) / max; + atomic_long_set(&per_cpu(cpu_freq_capacity, cpu), capacity); +} + +static int cpufreq_callback(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_freqs *freq = data; + int cpu = freq->cpu; + unsigned long max = atomic_long_read(&per_cpu(cpu_max_freq, cpu)); + + if (freq->flags & CPUFREQ_CONST_LOOPS) + return NOTIFY_OK; + + if (val == CPUFREQ_PRECHANGE) { + scale_freq_capacity(cpu, freq->new, max); + // TJK: fix tracing: trace_cpu_capacity(capacity_curr_of(cpu), cpu); + } + + return NOTIFY_OK; +} + +static struct notifier_block cpufreq_notifier = { + .notifier_call = cpufreq_callback, +}; + +static int cpufreq_policy_callback(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_policy *policy = data; + int i; + + if (val != CPUFREQ_NOTIFY) + return NOTIFY_OK; + + for_each_cpu(i, policy->cpus) { + scale_freq_capacity(i, policy->cur, policy->max); + atomic_long_set(&per_cpu(cpu_max_freq, i), policy->max); + } + + return NOTIFY_OK; +} + +static struct notifier_block cpufreq_policy_notifier = { + .notifier_call = cpufreq_policy_callback, +}; + +static int __init register_cpufreq_notifier(void) +{ + int ret; + + ret = cpufreq_register_notifier(&cpufreq_notifier, + CPUFREQ_TRANSITION_NOTIFIER); + if (ret) + return ret; + + return cpufreq_register_notifier(&cpufreq_policy_notifier, + CPUFREQ_POLICY_NOTIFIER); +} +core_initcall(register_cpufreq_notifier); +#endif + diff --git a/kernel/smpboot.c b/kernel/smpboot.c index 7020eecb398bd..8d590db6d9e2a 100644 --- a/kernel/smpboot.c +++ b/kernel/smpboot.c @@ -24,7 +24,7 @@ */ static DEFINE_PER_CPU(struct task_struct *, idle_threads); -struct task_struct * __cpuinit idle_thread_get(unsigned int cpu) +struct task_struct * idle_thread_get(unsigned int cpu) { struct task_struct *tsk = per_cpu(idle_threads, cpu); diff --git a/kernel/softirq.c b/kernel/softirq.c index 96a4c2828ce65..9de0f2730c080 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -712,7 +712,7 @@ static void takeover_tasklets(unsigned int cpu) } #endif /* CONFIG_HOTPLUG_CPU */ -static int __cpuinit cpu_callback(struct notifier_block *nfb, +static int cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { @@ -727,7 +727,7 @@ static int __cpuinit cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata cpu_nfb = { +static struct notifier_block cpu_nfb = { .notifier_call = cpu_callback }; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index e31e69332d5c8..4b248b705dd85 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -297,8 +297,8 @@ static struct ctl_table kern_table[] = { }, #endif { - .procname = "sched_wakeup_load_threshold", - .data = &sysctl_sched_wakeup_load_threshold, + .procname = "sched_capacity_margin", + .data = &sysctl_sched_capacity_margin, .maxlen = sizeof(unsigned int), .mode = 0644, .proc_handler = proc_dointvec, @@ -334,6 +334,7 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = sched_window_update_handler, }, +#ifdef TJK_HMP { .procname = "sched_heavy_task", .data = &sysctl_sched_heavy_task_pct, @@ -342,6 +343,7 @@ static struct ctl_table kern_table[] = { .proc_handler = sched_hmp_proc_update_handler, }, #endif +#endif #ifdef CONFIG_SCHED_HMP { .procname = "sched_account_wait_time", @@ -371,6 +373,7 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = sched_window_update_handler, }, +#ifdef TJK_HMP { .procname = "sched_small_task", .data = &sysctl_sched_small_task_pct, @@ -392,6 +395,7 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = sched_hmp_proc_update_handler, }, +#endif { .procname = "sched_spill_nr_run", .data = &sysctl_sched_spill_nr_run, @@ -400,6 +404,7 @@ static struct ctl_table kern_table[] = { .proc_handler = proc_dointvec_minmax, .extra1 = &zero, }, +#ifdef TJK_HMP { .procname = "sched_upmigrate", .data = &sysctl_sched_upmigrate_pct, @@ -435,6 +440,7 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = sched_hmp_proc_update_handler, }, +#endif { .procname = "sched_boost", .data = &sysctl_sched_boost, @@ -471,6 +477,20 @@ static struct ctl_table kern_table[] = { .extra1 = &min_sched_granularity_ns, .extra2 = &max_sched_granularity_ns, }, + { + .procname = "sched_is_big_little", + .data = &sysctl_sched_is_big_little, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "sched_sync_hint_enable", + .data = &sysctl_sched_sync_hint_enable, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, { .procname = "sched_wakeup_granularity_ns", .data = &sysctl_sched_wakeup_granularity, diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 228eba5ccc72e..c0e1b0e56d3e5 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -331,7 +331,7 @@ static int __init tick_nohz_full_setup(char *str) } __setup("nohz_full=", tick_nohz_full_setup); -static int __cpuinit tick_nohz_cpu_down_callback(struct notifier_block *nfb, +static int tick_nohz_cpu_down_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { diff --git a/kernel/timer.c b/kernel/timer.c index bbc99703765fb..0524f8104d418 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -1572,11 +1572,11 @@ signed long __sched schedule_timeout_uninterruptible(signed long timeout) } EXPORT_SYMBOL(schedule_timeout_uninterruptible); -static int __cpuinit init_timers_cpu(int cpu) +static int init_timers_cpu(int cpu) { int j; struct tvec_base *base; - static char __cpuinitdata tvec_base_done[NR_CPUS + 1]; + static char tvec_base_done[NR_CPUS + 1]; if (!tvec_base_done[cpu]) { static char boot_done; @@ -1659,7 +1659,7 @@ static void migrate_timer_list(struct tvec_base *new_base, struct list_head *hea } } -static void __cpuinit migrate_timers(int cpu) +static void migrate_timers(int cpu) { struct tvec_base *old_base; struct tvec_base *new_base; @@ -1692,7 +1692,7 @@ static void __cpuinit migrate_timers(int cpu) } #endif /* CONFIG_HOTPLUG_CPU */ -static int __cpuinit timer_cpu_notify(struct notifier_block *self, +static int timer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { long cpu = (long)hcpu; @@ -1717,7 +1717,7 @@ static int __cpuinit timer_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata timers_nb = { +static struct notifier_block timers_nb = { .notifier_call = timer_cpu_notify, }; diff --git a/kernel/workqueue.c b/kernel/workqueue.c index c67dc9c89f3b2..21d7ce5b3c0b0 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -4755,7 +4755,7 @@ static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu) * Workqueues should be brought up before normal priority CPU notifiers. * This will be registered high priority CPU notifier. */ -static int __cpuinit workqueue_cpu_up_callback(struct notifier_block *nfb, +static int workqueue_cpu_up_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { @@ -4808,7 +4808,7 @@ static int __cpuinit workqueue_cpu_up_callback(struct notifier_block *nfb, * Workqueues should be brought down after normal priority CPU notifiers. * This will be registered as low priority CPU notifier. */ -static int __cpuinit workqueue_cpu_down_callback(struct notifier_block *nfb, +static int workqueue_cpu_down_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { diff --git a/lib/earlycpio.c b/lib/earlycpio.c index 8078ef49cb793..7aa7ce250c94d 100644 --- a/lib/earlycpio.c +++ b/lib/earlycpio.c @@ -63,7 +63,7 @@ enum cpio_fields { * the match returned an empty filename string. */ -struct cpio_data __cpuinit find_cpio_data(const char *path, void *data, +struct cpio_data find_cpio_data(const char *path, void *data, size_t len, long *offset) { const size_t cpio_header_len = 8*C_NFIELDS - 2; diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c index ba6085d9c7411..867f2da6d697d 100644 --- a/lib/percpu_counter.c +++ b/lib/percpu_counter.c @@ -158,7 +158,7 @@ static void compute_batch_value(void) percpu_counter_batch = max(32, nr*2); } -static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb, +static int percpu_counter_hotcpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu) { #ifdef CONFIG_HOTPLUG_CPU diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 904244a74e43b..356caf075c918 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2565,7 +2565,7 @@ static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu) spin_unlock(&memcg->pcp_counter_lock); } -static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb, +static int memcg_cpu_hotplug_callback(struct notifier_block *nb, unsigned long action, void *hcpu) { diff --git a/mm/memory.c b/mm/memory.c index a3646c281a0f5..678bd985cfdd1 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4196,16 +4196,13 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, struct mm_struct *mm; int ret; - skip_cfs_throttle(1); mm = get_task_mm(tsk); if (!mm) { - skip_cfs_throttle(0); return 0; } ret = __access_remote_vm(tsk, mm, addr, buf, len, write); mmput(mm); - skip_cfs_throttle(0); return ret; } diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 7474cf43110cd..e0ec4b74f5ecd 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -1761,7 +1761,7 @@ void writeback_set_ratelimit(void) ratelimit_pages = 16; } -static int __cpuinit +static int ratelimit_handler(struct notifier_block *self, unsigned long action, void *hcpu) { @@ -1776,7 +1776,7 @@ ratelimit_handler(struct notifier_block *self, unsigned long action, } } -static struct notifier_block __cpuinitdata ratelimit_nb = { +static struct notifier_block ratelimit_nb = { .notifier_call = ratelimit_handler, .next = NULL, }; diff --git a/mm/slab.c b/mm/slab.c index bd88411595b9b..a1b22c71c666c 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -787,7 +787,7 @@ static void next_reap_node(void) * the CPUs getting into lockstep and contending for the global cache chain * lock. */ -static void __cpuinit start_cpu_timer(int cpu) +static void start_cpu_timer(int cpu) { struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu); @@ -1180,7 +1180,7 @@ static int init_cache_node_node(int node) return 0; } -static void __cpuinit cpuup_canceled(long cpu) +static void cpuup_canceled(long cpu) { struct kmem_cache *cachep; struct kmem_cache_node *n = NULL; @@ -1245,7 +1245,7 @@ static void __cpuinit cpuup_canceled(long cpu) } } -static int __cpuinit cpuup_prepare(long cpu) +static int cpuup_prepare(long cpu) { struct kmem_cache *cachep; struct kmem_cache_node *n = NULL; @@ -1328,7 +1328,7 @@ static int __cpuinit cpuup_prepare(long cpu) return -ENOMEM; } -static int __cpuinit cpuup_callback(struct notifier_block *nfb, +static int cpuup_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { long cpu = (long)hcpu; @@ -1384,7 +1384,7 @@ static int __cpuinit cpuup_callback(struct notifier_block *nfb, return notifier_from_errno(err); } -static struct notifier_block __cpuinitdata cpucache_notifier = { +static struct notifier_block cpucache_notifier = { &cpuup_callback, NULL, 0 }; diff --git a/mm/slub.c b/mm/slub.c index 411d812471437..6d3b32f081b4d 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -3769,7 +3769,7 @@ int __kmem_cache_create(struct kmem_cache *s, unsigned long flags) * Use the cpu notifier to insure that the cpu slabs are flushed when * necessary. */ -static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb, +static int slab_cpuup_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { long cpu = (long)hcpu; @@ -3795,7 +3795,7 @@ static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata slab_notifier = { +static struct notifier_block slab_notifier = { .notifier_call = slab_cpuup_callback }; diff --git a/mm/vmstat.c b/mm/vmstat.c index c4a665630d4de..8f3e015b42759 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1281,7 +1281,7 @@ static void vmstat_update(struct work_struct *w) round_jiffies_relative(sysctl_stat_interval)); } -static void __cpuinit start_cpu_timer(int cpu) +static void start_cpu_timer(int cpu) { struct delayed_work *work = &per_cpu(vmstat_work, cpu); @@ -1293,7 +1293,7 @@ static void __cpuinit start_cpu_timer(int cpu) * Use the cpu notifier to insure that the thresholds are recalculated * when necessary. */ -static int __cpuinit vmstat_cpuup_callback(struct notifier_block *nfb, +static int vmstat_cpuup_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { @@ -1325,7 +1325,7 @@ static int __cpuinit vmstat_cpuup_callback(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata vmstat_notifier = +static struct notifier_block vmstat_notifier = { &vmstat_cpuup_callback, NULL, 0 }; #endif diff --git a/net/core/flow.c b/net/core/flow.c index f74230f808675..9a2151f5d5935 100644 --- a/net/core/flow.c +++ b/net/core/flow.c @@ -403,7 +403,7 @@ void flow_cache_flush_deferred(void) schedule_work(&flow_cache_flush_work); } -static int __cpuinit flow_cache_cpu_prepare(struct flow_cache *fc, int cpu) +static int flow_cache_cpu_prepare(struct flow_cache *fc, int cpu) { struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu); size_t sz = sizeof(struct hlist_head) * flow_cache_hash_size(fc); @@ -421,7 +421,7 @@ static int __cpuinit flow_cache_cpu_prepare(struct flow_cache *fc, int cpu) return 0; } -static int __cpuinit flow_cache_cpu(struct notifier_block *nfb, +static int flow_cache_cpu(struct notifier_block *nfb, unsigned long action, void *hcpu) { diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c index 4fe76ff214c2a..cd5b8ec9be045 100644 --- a/net/iucv/iucv.c +++ b/net/iucv/iucv.c @@ -621,7 +621,7 @@ static void iucv_disable(void) put_online_cpus(); } -static int __cpuinit iucv_cpu_notify(struct notifier_block *self, +static int iucv_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { cpumask_t cpumask; diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 2fa73ee7942f3..dd907ee6e2ac9 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -868,23 +868,23 @@ static void check_section(const char *modname, struct elf_info *elf, #define ALL_INIT_DATA_SECTIONS \ ".init.setup$", ".init.rodata$", \ - ".cpuinit.rodata$", ".meminit.rodata$", \ - ".init.data$", ".cpuinit.data$", ".meminit.data$" + ".meminit.rodata$", \ + ".init.data$", ".meminit.data$" #define ALL_EXIT_DATA_SECTIONS \ - ".exit.data$", ".cpuexit.data$", ".memexit.data$" + ".exit.data$", ".memexit.data$" #define ALL_INIT_TEXT_SECTIONS \ - ".init.text$", ".cpuinit.text$", ".meminit.text$" + ".init.text$", ".meminit.text$" #define ALL_EXIT_TEXT_SECTIONS \ - ".exit.text$", ".cpuexit.text$", ".memexit.text$" + ".exit.text$", ".memexit.text$" #define ALL_PCI_INIT_SECTIONS \ ".pci_fixup_early$", ".pci_fixup_header$", ".pci_fixup_final$", \ ".pci_fixup_enable$", ".pci_fixup_resume$", \ ".pci_fixup_resume_early$", ".pci_fixup_suspend$" -#define ALL_XXXINIT_SECTIONS CPU_INIT_SECTIONS, MEM_INIT_SECTIONS -#define ALL_XXXEXIT_SECTIONS CPU_EXIT_SECTIONS, MEM_EXIT_SECTIONS +#define ALL_XXXINIT_SECTIONS MEM_INIT_SECTIONS +#define ALL_XXXEXIT_SECTIONS MEM_EXIT_SECTIONS #define ALL_INIT_SECTIONS INIT_SECTIONS, ALL_XXXINIT_SECTIONS #define ALL_EXIT_SECTIONS EXIT_SECTIONS, ALL_XXXEXIT_SECTIONS @@ -893,11 +893,9 @@ static void check_section(const char *modname, struct elf_info *elf, #define TEXT_SECTIONS ".text$" #define INIT_SECTIONS ".init.*" -#define CPU_INIT_SECTIONS ".cpuinit.*" #define MEM_INIT_SECTIONS ".meminit.*" #define EXIT_SECTIONS ".exit.*" -#define CPU_EXIT_SECTIONS ".cpuexit.*" #define MEM_EXIT_SECTIONS ".memexit.*" /* init data sections */ @@ -985,48 +983,20 @@ const struct sectioncheck sectioncheck[] = { .mismatch = DATA_TO_ANY_EXIT, .symbol_white_list = { DEFAULT_SYMBOL_WHITE_LIST, NULL }, }, -/* Do not reference init code/data from cpuinit/meminit code/data */ +/* Do not reference init code/data from meminit code/data */ { .fromsec = { ALL_XXXINIT_SECTIONS, NULL }, .tosec = { INIT_SECTIONS, NULL }, .mismatch = XXXINIT_TO_SOME_INIT, .symbol_white_list = { DEFAULT_SYMBOL_WHITE_LIST, NULL }, }, -/* Do not reference cpuinit code/data from meminit code/data */ -{ - .fromsec = { MEM_INIT_SECTIONS, NULL }, - .tosec = { CPU_INIT_SECTIONS, NULL }, - .mismatch = XXXINIT_TO_SOME_INIT, - .symbol_white_list = { DEFAULT_SYMBOL_WHITE_LIST, NULL }, -}, -/* Do not reference meminit code/data from cpuinit code/data */ -{ - .fromsec = { CPU_INIT_SECTIONS, NULL }, - .tosec = { MEM_INIT_SECTIONS, NULL }, - .mismatch = XXXINIT_TO_SOME_INIT, - .symbol_white_list = { DEFAULT_SYMBOL_WHITE_LIST, NULL }, -}, -/* Do not reference exit code/data from cpuexit/memexit code/data */ +/* Do not reference exit code/data from memexit code/data */ { .fromsec = { ALL_XXXEXIT_SECTIONS, NULL }, .tosec = { EXIT_SECTIONS, NULL }, .mismatch = XXXEXIT_TO_SOME_EXIT, .symbol_white_list = { DEFAULT_SYMBOL_WHITE_LIST, NULL }, }, -/* Do not reference cpuexit code/data from memexit code/data */ -{ - .fromsec = { MEM_EXIT_SECTIONS, NULL }, - .tosec = { CPU_EXIT_SECTIONS, NULL }, - .mismatch = XXXEXIT_TO_SOME_EXIT, - .symbol_white_list = { DEFAULT_SYMBOL_WHITE_LIST, NULL }, -}, -/* Do not reference memexit code/data from cpuexit code/data */ -{ - .fromsec = { CPU_EXIT_SECTIONS, NULL }, - .tosec = { MEM_EXIT_SECTIONS, NULL }, - .mismatch = XXXEXIT_TO_SOME_EXIT, - .symbol_white_list = { DEFAULT_SYMBOL_WHITE_LIST, NULL }, -}, /* Do not use exit code/data from init code */ { .fromsec = { ALL_INIT_SECTIONS, NULL }, @@ -1095,7 +1065,7 @@ static const struct sectioncheck *section_mismatch( * Pattern 2: * Many drivers utilise a *driver container with references to * add, remove, probe functions etc. - * These functions may often be marked __cpuinit and we do not want to + * These functions may often be marked and we do not want to * warn here. * the pattern is identified by: * tosec = init or exit section @@ -1255,7 +1225,6 @@ static Elf_Sym *find_elf_symbol2(struct elf_info *elf, Elf_Addr addr, /* * Convert a section name to the function/data attribute * .init.text => __init - * .cpuinit.data => __cpudata * .memexitconst => __memconst * etc. * diff --git a/security/capability.c b/security/capability.c index 1728d4e375db5..6e4fc776badf1 100644 --- a/security/capability.c +++ b/security/capability.c @@ -12,6 +12,26 @@ #include +static int cap_binder_set_context_mgr(struct task_struct *mgr) +{ + return 0; +} + +static int cap_binder_transaction(struct task_struct *from, struct task_struct *to) +{ + return 0; +} + +static int cap_binder_transfer_binder(struct task_struct *from, struct task_struct *to) +{ + return 0; +} + +static int cap_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) +{ + return 0; +} + static int cap_syslog(int type) { return 0; @@ -903,6 +923,10 @@ static void cap_audit_rule_free(void *lsmrule) void __init security_fixup_ops(struct security_operations *ops) { + set_to_cap_if_null(ops, binder_set_context_mgr); + set_to_cap_if_null(ops, binder_transaction); + set_to_cap_if_null(ops, binder_transfer_binder); + set_to_cap_if_null(ops, binder_transfer_file); set_to_cap_if_null(ops, ptrace_access_check); set_to_cap_if_null(ops, ptrace_traceme); set_to_cap_if_null(ops, capget); diff --git a/security/security.c b/security/security.c index 12380e7eb76f4..ea34b97abe44b 100644 --- a/security/security.c +++ b/security/security.c @@ -134,6 +134,26 @@ int __init register_security(struct security_operations *ops) /* Security operations */ +int security_binder_set_context_mgr(struct task_struct *mgr) +{ + return security_ops->binder_set_context_mgr(mgr); +} + +int security_binder_transaction(struct task_struct *from, struct task_struct *to) +{ + return security_ops->binder_transaction(from, to); +} + +int security_binder_transfer_binder(struct task_struct *from, struct task_struct *to) +{ + return security_ops->binder_transfer_binder(from, to); +} + +int security_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) +{ + return security_ops->binder_transfer_file(from, to, file); +} + int security_ptrace_access_check(struct task_struct *child, unsigned int mode) { #ifdef CONFIG_SECURITY_YAMA_STACKED diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 29d96007f9d56..626533313cf57 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -1871,6 +1871,67 @@ static inline u32 open_file_to_av(struct file *file) /* Hook functions begin here. */ +static int selinux_binder_set_context_mgr(struct task_struct *mgr) +{ + u32 mysid = current_sid(); + u32 mgrsid = task_sid(mgr); + + return avc_has_perm(mysid, mgrsid, SECCLASS_BINDER, BINDER__SET_CONTEXT_MGR, NULL); +} + +static int selinux_binder_transaction(struct task_struct *from, struct task_struct *to) +{ + u32 mysid = current_sid(); + u32 fromsid = task_sid(from); + u32 tosid = task_sid(to); + int rc; + + if (mysid != fromsid) { + rc = avc_has_perm(mysid, fromsid, SECCLASS_BINDER, BINDER__IMPERSONATE, NULL); + if (rc) + return rc; + } + + return avc_has_perm(fromsid, tosid, SECCLASS_BINDER, BINDER__CALL, NULL); +} + +static int selinux_binder_transfer_binder(struct task_struct *from, struct task_struct *to) +{ + u32 fromsid = task_sid(from); + u32 tosid = task_sid(to); + return avc_has_perm(fromsid, tosid, SECCLASS_BINDER, BINDER__TRANSFER, NULL); +} + +static int selinux_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) +{ + u32 sid = task_sid(to); + struct file_security_struct *fsec = file->f_security; + struct inode *inode = file->f_path.dentry->d_inode; + struct inode_security_struct *isec = inode->i_security; + struct common_audit_data ad; + struct selinux_audit_data sad = {0,}; + int rc; + + ad.type = LSM_AUDIT_DATA_PATH; + ad.u.path = file->f_path; + ad.selinux_audit_data = &sad; + + if (sid != fsec->sid) { + rc = avc_has_perm(sid, fsec->sid, + SECCLASS_FD, + FD__USE, + &ad); + if (rc) + return rc; + } + + if (unlikely(IS_PRIVATE(inode))) + return 0; + + return avc_has_perm(sid, isec->sid, isec->sclass, file_to_av(file), + &ad); +} + static int selinux_ptrace_access_check(struct task_struct *child, unsigned int mode) { @@ -5717,6 +5778,11 @@ static int selinux_key_getsecurity(struct key *key, char **_buffer) static struct security_operations selinux_ops = { .name = "selinux", + .binder_set_context_mgr = selinux_binder_set_context_mgr, + .binder_transaction = selinux_binder_transaction, + .binder_transfer_binder = selinux_binder_transfer_binder, + .binder_transfer_file = selinux_binder_transfer_file, + .ptrace_access_check = selinux_ptrace_access_check, .ptrace_traceme = selinux_ptrace_traceme, .capget = selinux_capget, diff --git a/security/selinux/include/classmap.h b/security/selinux/include/classmap.h index 14d04e63b1f0e..c32ff7bde81ab 100644 --- a/security/selinux/include/classmap.h +++ b/security/selinux/include/classmap.h @@ -151,5 +151,6 @@ struct security_class_mapping secclass_map[] = { { "kernel_service", { "use_as_override", "create_files_as", NULL } }, { "tun_socket", { COMMON_SOCK_PERMS, "attach_queue", NULL } }, + { "binder", { "impersonate", "call", "set_context_mgr", "transfer", NULL } }, { NULL } };