kernel/sched/isolation.c

   1 /*
   2  *  Housekeeping management. Manage the targets for routine code that can run on
   3  *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
   4  *
   5  * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
   6  * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
   7  *
   8  */
   9 #include "sched.h"
  10
  11 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
  12 EXPORT_SYMBOL_GPL(housekeeping_overridden);
  13 static cpumask_var_t housekeeping_mask;
  14 static unsigned int housekeeping_flags;
  15
  16 int housekeeping_any_cpu(enum hk_flags flags)
  17 {
  18         if (static_branch_unlikely(&housekeeping_overridden))
  19                 if (housekeeping_flags & flags)
  20                         return cpumask_any_and(housekeeping_mask, cpu_online_mask);
  21         return smp_processor_id();
  22 }
  23 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
  24
  25 const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
  26 {
  27         if (static_branch_unlikely(&housekeeping_overridden))
  28                 if (housekeeping_flags & flags)
  29                         return housekeeping_mask;
  30         return cpu_possible_mask;
  31 }
  32 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
  33
  34 void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
  35 {
  36         if (static_branch_unlikely(&housekeeping_overridden))
  37                 if (housekeeping_flags & flags)
  38                         set_cpus_allowed_ptr(t, housekeeping_mask);
  39 }
  40 EXPORT_SYMBOL_GPL(housekeeping_affine);
  41
  42 bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
  43 {
  44         if (static_branch_unlikely(&housekeeping_overridden))
  45                 if (housekeeping_flags & flags)
  46                         return cpumask_test_cpu(cpu, housekeeping_mask);
  47         return true;
  48 }
  49 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
  50
  51 void __init housekeeping_init(void)
  52 {
  53         if (!housekeeping_flags)
  54                 return;
  55
  56         static_branch_enable(&housekeeping_overridden);
  57
  58         if (housekeeping_flags & HK_FLAG_TICK)
  59                 sched_tick_offload_init();
  60
  61         /* We need at least one CPU to handle housekeeping work */
  62         WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
  63 }
  64
  65 static int __init housekeeping_setup(char *str, enum hk_flags flags)
  66 {
  67         cpumask_var_t non_housekeeping_mask;
  68         cpumask_var_t tmp;
  69         int err;
  70
  71         alloc_bootmem_cpumask_var(&non_housekeeping_mask);
  72         err = cpulist_parse(str, non_housekeeping_mask);
  73         if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
  74                 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
  75                 free_bootmem_cpumask_var(non_housekeeping_mask);
  76                 return 0;
  77         }
  78
  79         alloc_bootmem_cpumask_var(&tmp);
  80         if (!housekeeping_flags) {
  81                 alloc_bootmem_cpumask_var(&housekeeping_mask);
  82                 cpumask_andnot(housekeeping_mask,
  83                                cpu_possible_mask, non_housekeeping_mask);
  84
  85                 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
  86                 if (cpumask_empty(tmp)) {
  87                         pr_warn("Housekeeping: must include one present CPU, "
  88                                 "using boot CPU:%d\n", smp_processor_id());
  89                         __cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
  90                         __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
  91                 }
  92         } else {
  93                 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
  94                 if (cpumask_empty(tmp))
  95                         __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
  96                 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
  97                 if (!cpumask_equal(tmp, housekeeping_mask)) {
  98                         pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
  99                         free_bootmem_cpumask_var(tmp);
 100                         free_bootmem_cpumask_var(non_housekeeping_mask);
 101                         return 0;
 102                 }
 103         }
 104         free_bootmem_cpumask_var(tmp);
 105
 106         if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
 107                 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
 108                         tick_nohz_full_setup(non_housekeeping_mask);
 109                 } else {
 110                         pr_warn("Housekeeping: nohz unsupported."
 111                                 " Build with CONFIG_NO_HZ_FULL\n");
 112                         free_bootmem_cpumask_var(non_housekeeping_mask);
 113                         return 0;
 114                 }
 115         }
 116
 117         housekeeping_flags |= flags;
 118
 119         free_bootmem_cpumask_var(non_housekeeping_mask);
 120
 121         return 1;
 122 }
 123
 124 static int __init housekeeping_nohz_full_setup(char *str)
 125 {
 126         unsigned int flags;
 127
 128         flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC;
 129
 130         return housekeeping_setup(str, flags);
 131 }
 132 __setup("nohz_full=", housekeeping_nohz_full_setup);
 133
 134 static int __init housekeeping_isolcpus_setup(char *str)
 135 {
 136         unsigned int flags = 0;
 137
 138         while (isalpha(*str)) {
 139                 if (!strncmp(str, "nohz,", 5)) {
 140                         str += 5;
 141                         flags |= HK_FLAG_TICK;
 142                         continue;
 143                 }
 144
 145                 if (!strncmp(str, "domain,", 7)) {
 146                         str += 7;
 147                         flags |= HK_FLAG_DOMAIN;
 148                         continue;
 149                 }
 150
 151                 pr_warn("isolcpus: Error, unknown flag\n");
 152                 return 0;
 153         }
 154
 155         /* Default behaviour for isolcpus without flags */
 156         if (!flags)
 157                 flags |= HK_FLAG_DOMAIN;
 158
 159         return housekeeping_setup(str, flags);
 160 }
 161 __setup("isolcpus=", housekeeping_isolcpus_setup);