1 // SPDX-License-Identifier: GPL-2.0-only
3 * Housekeeping management. Manage the targets for routine code that can run on
4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work.
6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
12 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
13 EXPORT_SYMBOL_GPL(housekeeping_overridden);
14 static cpumask_var_t housekeeping_mask;
15 static unsigned int housekeeping_flags;
17 int housekeeping_any_cpu(enum hk_flags flags)
19 if (static_branch_unlikely(&housekeeping_overridden))
20 if (housekeeping_flags & flags)
21 return cpumask_any_and(housekeeping_mask, cpu_online_mask);
22 return smp_processor_id();
24 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
26 const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
28 if (static_branch_unlikely(&housekeeping_overridden))
29 if (housekeeping_flags & flags)
30 return housekeeping_mask;
31 return cpu_possible_mask;
33 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
35 void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
37 if (static_branch_unlikely(&housekeeping_overridden))
38 if (housekeeping_flags & flags)
39 set_cpus_allowed_ptr(t, housekeeping_mask);
41 EXPORT_SYMBOL_GPL(housekeeping_affine);
43 bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
45 if (static_branch_unlikely(&housekeeping_overridden))
46 if (housekeeping_flags & flags)
47 return cpumask_test_cpu(cpu, housekeeping_mask);
50 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
52 void __init housekeeping_init(void)
54 if (!housekeeping_flags)
57 static_branch_enable(&housekeeping_overridden);
59 if (housekeeping_flags & HK_FLAG_TICK)
60 sched_tick_offload_init();
62 /* We need at least one CPU to handle housekeeping work */
63 WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
66 static int __init housekeeping_setup(char *str, enum hk_flags flags)
68 cpumask_var_t non_housekeeping_mask;
72 alloc_bootmem_cpumask_var(&non_housekeeping_mask);
73 err = cpulist_parse(str, non_housekeeping_mask);
74 if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
75 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
76 free_bootmem_cpumask_var(non_housekeeping_mask);
80 alloc_bootmem_cpumask_var(&tmp);
81 if (!housekeeping_flags) {
82 alloc_bootmem_cpumask_var(&housekeeping_mask);
83 cpumask_andnot(housekeeping_mask,
84 cpu_possible_mask, non_housekeeping_mask);
86 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
87 if (cpumask_empty(tmp)) {
88 pr_warn("Housekeeping: must include one present CPU, "
89 "using boot CPU:%d\n", smp_processor_id());
90 __cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
91 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
94 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
95 if (cpumask_empty(tmp))
96 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
97 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
98 if (!cpumask_equal(tmp, housekeeping_mask)) {
99 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
100 free_bootmem_cpumask_var(tmp);
101 free_bootmem_cpumask_var(non_housekeeping_mask);
105 free_bootmem_cpumask_var(tmp);
107 if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
108 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
109 tick_nohz_full_setup(non_housekeeping_mask);
111 pr_warn("Housekeeping: nohz unsupported."
112 " Build with CONFIG_NO_HZ_FULL\n");
113 free_bootmem_cpumask_var(non_housekeeping_mask);
118 housekeeping_flags |= flags;
120 free_bootmem_cpumask_var(non_housekeeping_mask);
125 static int __init housekeeping_nohz_full_setup(char *str)
129 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC;
131 return housekeeping_setup(str, flags);
133 __setup("nohz_full=", housekeeping_nohz_full_setup);
135 static int __init housekeeping_isolcpus_setup(char *str)
137 unsigned int flags = 0;
139 while (isalpha(*str)) {
140 if (!strncmp(str, "nohz,", 5)) {
142 flags |= HK_FLAG_TICK;
146 if (!strncmp(str, "domain,", 7)) {
148 flags |= HK_FLAG_DOMAIN;
152 pr_warn("isolcpus: Error, unknown flag\n");
156 /* Default behaviour for isolcpus without flags */
158 flags |= HK_FLAG_DOMAIN;
160 return housekeeping_setup(str, flags);
162 __setup("isolcpus=", housekeeping_isolcpus_setup);