arch/xtensa/kernel/smp.c

   1 /*
   2  * Xtensa SMP support functions.
   3  *
   4  * This file is subject to the terms and conditions of the GNU General Public
   5  * License.  See the file "COPYING" in the main directory of this archive
   6  * for more details.
   7  *
   8  * Copyright (C) 2008 - 2013 Tensilica Inc.
   9  *
  10  * Chris Zankel <chris@zankel.net>
  11  * Joe Taylor <joe@tensilica.com>
  12  * Pete Delaney <piet@tensilica.com
  13  */
  14
  15 #include <linux/cpu.h>
  16 #include <linux/cpumask.h>
  17 #include <linux/delay.h>
  18 #include <linux/init.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/irqdomain.h>
  21 #include <linux/irq.h>
  22 #include <linux/kdebug.h>
  23 #include <linux/module.h>
  24 #include <linux/sched/hotplug.h>
  25 #include <linux/reboot.h>
  26 #include <linux/seq_file.h>
  27 #include <linux/smp.h>
  28 #include <linux/thread_info.h>
  29
  30 #include <asm/cacheflush.h>
  31 #include <asm/kdebug.h>
  32 #include <asm/mmu_context.h>
  33 #include <asm/mxregs.h>
  34 #include <asm/platform.h>
  35 #include <asm/tlbflush.h>
  36 #include <asm/traps.h>
  37
  38 #ifdef CONFIG_SMP
  39 # if XCHAL_HAVE_S32C1I == 0
  40 #  error "The S32C1I option is required for SMP."
  41 # endif
  42 #endif
  43
  44 static void system_invalidate_dcache_range(unsigned long start,
  45                 unsigned long size);
  46 static void system_flush_invalidate_dcache_range(unsigned long start,
  47                 unsigned long size);
  48
  49 /* IPI (Inter Process Interrupt) */
  50
  51 #define IPI_IRQ 0
  52
  53 static irqreturn_t ipi_interrupt(int irq, void *dev_id);
  54 static struct irqaction ipi_irqaction = {
  55         .handler =      ipi_interrupt,
  56         .flags =        IRQF_PERCPU,
  57         .name =         "ipi",
  58 };
  59
  60 void ipi_init(void)
  61 {
  62         unsigned irq = irq_create_mapping(NULL, IPI_IRQ);
  63         setup_irq(irq, &ipi_irqaction);
  64 }
  65
  66 static inline unsigned int get_core_count(void)
  67 {
  68         /* Bits 18..21 of SYSCFGID contain the core count minus 1. */
  69         unsigned int syscfgid = get_er(SYSCFGID);
  70         return ((syscfgid >> 18) & 0xf) + 1;
  71 }
  72
  73 static inline int get_core_id(void)
  74 {
  75         /* Bits 0...18 of SYSCFGID contain the core id  */
  76         unsigned int core_id = get_er(SYSCFGID);
  77         return core_id & 0x3fff;
  78 }
  79
  80 void __init smp_prepare_cpus(unsigned int max_cpus)
  81 {
  82         unsigned i;
  83
  84         for (i = 0; i < max_cpus; ++i)
  85                 set_cpu_present(i, true);
  86 }
  87
  88 void __init smp_init_cpus(void)
  89 {
  90         unsigned i;
  91         unsigned int ncpus = get_core_count();
  92         unsigned int core_id = get_core_id();
  93
  94         pr_info("%s: Core Count = %d\n", __func__, ncpus);
  95         pr_info("%s: Core Id = %d\n", __func__, core_id);
  96
  97         for (i = 0; i < ncpus; ++i)
  98                 set_cpu_possible(i, true);
  99 }
 100
 101 void __init smp_prepare_boot_cpu(void)
 102 {
 103         unsigned int cpu = smp_processor_id();
 104         BUG_ON(cpu != 0);
 105         cpu_asid_cache(cpu) = ASID_USER_FIRST;
 106 }
 107
 108 void __init smp_cpus_done(unsigned int max_cpus)
 109 {
 110 }
 111
 112 static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */
 113 static DECLARE_COMPLETION(cpu_running);
 114
 115 void secondary_start_kernel(void)
 116 {
 117         struct mm_struct *mm = &init_mm;
 118         unsigned int cpu = smp_processor_id();
 119
 120         init_mmu();
 121
 122 #ifdef CONFIG_DEBUG_KERNEL
 123         if (boot_secondary_processors == 0) {
 124                 pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n",
 125                         __func__, boot_secondary_processors, cpu);
 126                 for (;;)
 127                         __asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL));
 128         }
 129
 130         pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n",
 131                 __func__, boot_secondary_processors, cpu);
 132 #endif
 133         /* Init EXCSAVE1 */
 134
 135         secondary_trap_init();
 136
 137         /* All kernel threads share the same mm context. */
 138
 139         mmget(mm);
 140         mmgrab(mm);
 141         current->active_mm = mm;
 142         cpumask_set_cpu(cpu, mm_cpumask(mm));
 143         enter_lazy_tlb(mm, current);
 144
 145         preempt_disable();
 146         trace_hardirqs_off();
 147
 148         calibrate_delay();
 149
 150         notify_cpu_starting(cpu);
 151
 152         secondary_init_irq();
 153         local_timer_setup(cpu);
 154
 155         set_cpu_online(cpu, true);
 156
 157         local_irq_enable();
 158
 159         complete(&cpu_running);
 160
 161         cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
 162 }
 163
 164 static void mx_cpu_start(void *p)
 165 {
 166         unsigned cpu = (unsigned)p;
 167         unsigned long run_stall_mask = get_er(MPSCORE);
 168
 169         set_er(run_stall_mask & ~(1u << cpu), MPSCORE);
 170         pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
 171                         __func__, cpu, run_stall_mask, get_er(MPSCORE));
 172 }
 173
 174 static void mx_cpu_stop(void *p)
 175 {
 176         unsigned cpu = (unsigned)p;
 177         unsigned long run_stall_mask = get_er(MPSCORE);
 178
 179         set_er(run_stall_mask | (1u << cpu), MPSCORE);
 180         pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
 181                         __func__, cpu, run_stall_mask, get_er(MPSCORE));
 182 }
 183
 184 #ifdef CONFIG_HOTPLUG_CPU
 185 unsigned long cpu_start_id __cacheline_aligned;
 186 #endif
 187 unsigned long cpu_start_ccount;
 188
 189 static int boot_secondary(unsigned int cpu, struct task_struct *ts)
 190 {
 191         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
 192         unsigned long ccount;
 193         int i;
 194
 195 #ifdef CONFIG_HOTPLUG_CPU
 196         cpu_start_id = cpu;
 197         system_flush_invalidate_dcache_range(
 198                         (unsigned long)&cpu_start_id, sizeof(cpu_start_id));
 199 #endif
 200         smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1);
 201
 202         for (i = 0; i < 2; ++i) {
 203                 do
 204                         ccount = get_ccount();
 205                 while (!ccount);
 206
 207                 cpu_start_ccount = ccount;
 208
 209                 while (time_before(jiffies, timeout)) {
 210                         mb();
 211                         if (!cpu_start_ccount)
 212                                 break;
 213                 }
 214
 215                 if (cpu_start_ccount) {
 216                         smp_call_function_single(0, mx_cpu_stop,
 217                                         (void *)cpu, 1);
 218                         cpu_start_ccount = 0;
 219                         return -EIO;
 220                 }
 221         }
 222         return 0;
 223 }
 224
 225 int __cpu_up(unsigned int cpu, struct task_struct *idle)
 226 {
 227         int ret = 0;
 228
 229         if (cpu_asid_cache(cpu) == 0)
 230                 cpu_asid_cache(cpu) = ASID_USER_FIRST;
 231
 232         start_info.stack = (unsigned long)task_pt_regs(idle);
 233         wmb();
 234
 235         pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n",
 236                         __func__, cpu, idle, start_info.stack);
 237
 238         ret = boot_secondary(cpu, idle);
 239         if (ret == 0) {
 240                 wait_for_completion_timeout(&cpu_running,
 241                                 msecs_to_jiffies(1000));
 242                 if (!cpu_online(cpu))
 243                         ret = -EIO;
 244         }
 245
 246         if (ret)
 247                 pr_err("CPU %u failed to boot\n", cpu);
 248
 249         return ret;
 250 }
 251
 252 #ifdef CONFIG_HOTPLUG_CPU
 253
 254 /*
 255  * __cpu_disable runs on the processor to be shutdown.
 256  */
 257 int __cpu_disable(void)
 258 {
 259         unsigned int cpu = smp_processor_id();
 260
 261         /*
 262          * Take this CPU offline.  Once we clear this, we can't return,
 263          * and we must not schedule until we're ready to give up the cpu.
 264          */
 265         set_cpu_online(cpu, false);
 266
 267         /*
 268          * OK - migrate IRQs away from this CPU
 269          */
 270         migrate_irqs();
 271
 272         /*
 273          * Flush user cache and TLB mappings, and then remove this CPU
 274          * from the vm mask set of all processes.
 275          */
 276         local_flush_cache_all();
 277         local_flush_tlb_all();
 278         invalidate_page_directory();
 279
 280         clear_tasks_mm_cpumask(cpu);
 281
 282         return 0;
 283 }
 284
 285 static void platform_cpu_kill(unsigned int cpu)
 286 {
 287         smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true);
 288 }
 289
 290 /*
 291  * called on the thread which is asking for a CPU to be shutdown -
 292  * waits until shutdown has completed, or it is timed out.
 293  */
 294 void __cpu_die(unsigned int cpu)
 295 {
 296         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
 297         while (time_before(jiffies, timeout)) {
 298                 system_invalidate_dcache_range((unsigned long)&cpu_start_id,
 299                                 sizeof(cpu_start_id));
 300                 if (cpu_start_id == -cpu) {
 301                         platform_cpu_kill(cpu);
 302                         return;
 303                 }
 304         }
 305         pr_err("CPU%u: unable to kill\n", cpu);
 306 }
 307
 308 void arch_cpu_idle_dead(void)
 309 {
 310         cpu_die();
 311 }
 312 /*
 313  * Called from the idle thread for the CPU which has been shutdown.
 314  *
 315  * Note that we disable IRQs here, but do not re-enable them
 316  * before returning to the caller. This is also the behaviour
 317  * of the other hotplug-cpu capable cores, so presumably coming
 318  * out of idle fixes this.
 319  */
 320 void __ref cpu_die(void)
 321 {
 322         idle_task_exit();
 323         local_irq_disable();
 324         __asm__ __volatile__(
 325                         "       movi    a2, cpu_restart\n"
 326                         "       jx      a2\n");
 327 }
 328
 329 #endif /* CONFIG_HOTPLUG_CPU */
 330
 331 enum ipi_msg_type {
 332         IPI_RESCHEDULE = 0,
 333         IPI_CALL_FUNC,
 334         IPI_CPU_STOP,
 335         IPI_MAX
 336 };
 337
 338 static const struct {
 339         const char *short_text;
 340         const char *long_text;
 341 } ipi_text[] = {
 342         { .short_text = "RES", .long_text = "Rescheduling interrupts" },
 343         { .short_text = "CAL", .long_text = "Function call interrupts" },
 344         { .short_text = "DIE", .long_text = "CPU shutdown interrupts" },
 345 };
 346
 347 struct ipi_data {
 348         unsigned long ipi_count[IPI_MAX];
 349 };
 350
 351 static DEFINE_PER_CPU(struct ipi_data, ipi_data);
 352
 353 static void send_ipi_message(const struct cpumask *callmask,
 354                 enum ipi_msg_type msg_id)
 355 {
 356         int index;
 357         unsigned long mask = 0;
 358
 359         for_each_cpu(index, callmask)
 360                 if (index != smp_processor_id())
 361                         mask |= 1 << index;
 362
 363         set_er(mask, MIPISET(msg_id));
 364 }
 365
 366 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 367 {
 368         send_ipi_message(mask, IPI_CALL_FUNC);
 369 }
 370
 371 void arch_send_call_function_single_ipi(int cpu)
 372 {
 373         send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
 374 }
 375
 376 void smp_send_reschedule(int cpu)
 377 {
 378         send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
 379 }
 380
 381 void smp_send_stop(void)
 382 {
 383         struct cpumask targets;
 384
 385         cpumask_copy(&targets, cpu_online_mask);
 386         cpumask_clear_cpu(smp_processor_id(), &targets);
 387         send_ipi_message(&targets, IPI_CPU_STOP);
 388 }
 389
 390 static void ipi_cpu_stop(unsigned int cpu)
 391 {
 392         set_cpu_online(cpu, false);
 393         machine_halt();
 394 }
 395
 396 irqreturn_t ipi_interrupt(int irq, void *dev_id)
 397 {
 398         unsigned int cpu = smp_processor_id();
 399         struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
 400         unsigned int msg;
 401         unsigned i;
 402
 403         msg = get_er(MIPICAUSE(cpu));
 404         for (i = 0; i < IPI_MAX; i++)
 405                 if (msg & (1 << i)) {
 406                         set_er(1 << i, MIPICAUSE(cpu));
 407                         ++ipi->ipi_count[i];
 408                 }
 409
 410         if (msg & (1 << IPI_RESCHEDULE))
 411                 scheduler_ipi();
 412         if (msg & (1 << IPI_CALL_FUNC))
 413                 generic_smp_call_function_interrupt();
 414         if (msg & (1 << IPI_CPU_STOP))
 415                 ipi_cpu_stop(cpu);
 416
 417         return IRQ_HANDLED;
 418 }
 419
 420 void show_ipi_list(struct seq_file *p, int prec)
 421 {
 422         unsigned int cpu;
 423         unsigned i;
 424
 425         for (i = 0; i < IPI_MAX; ++i) {
 426                 seq_printf(p, "%*s:", prec, ipi_text[i].short_text);
 427                 for_each_online_cpu(cpu)
 428                         seq_printf(p, " %10lu",
 429                                         per_cpu(ipi_data, cpu).ipi_count[i]);
 430                 seq_printf(p, "   %s\n", ipi_text[i].long_text);
 431         }
 432 }
 433
 434 int setup_profiling_timer(unsigned int multiplier)
 435 {
 436         pr_debug("setup_profiling_timer %d\n", multiplier);
 437         return 0;
 438 }
 439
 440 /* TLB flush functions */
 441
 442 struct flush_data {
 443         struct vm_area_struct *vma;
 444         unsigned long addr1;
 445         unsigned long addr2;
 446 };
 447
 448 static void ipi_flush_tlb_all(void *arg)
 449 {
 450         local_flush_tlb_all();
 451 }
 452
 453 void flush_tlb_all(void)
 454 {
 455         on_each_cpu(ipi_flush_tlb_all, NULL, 1);
 456 }
 457
 458 static void ipi_flush_tlb_mm(void *arg)
 459 {
 460         local_flush_tlb_mm(arg);
 461 }
 462
 463 void flush_tlb_mm(struct mm_struct *mm)
 464 {
 465         on_each_cpu(ipi_flush_tlb_mm, mm, 1);
 466 }
 467
 468 static void ipi_flush_tlb_page(void *arg)
 469 {
 470         struct flush_data *fd = arg;
 471         local_flush_tlb_page(fd->vma, fd->addr1);
 472 }
 473
 474 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
 475 {
 476         struct flush_data fd = {
 477                 .vma = vma,
 478                 .addr1 = addr,
 479         };
 480         on_each_cpu(ipi_flush_tlb_page, &fd, 1);
 481 }
 482
 483 static void ipi_flush_tlb_range(void *arg)
 484 {
 485         struct flush_data *fd = arg;
 486         local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
 487 }
 488
 489 void flush_tlb_range(struct vm_area_struct *vma,
 490                      unsigned long start, unsigned long end)
 491 {
 492         struct flush_data fd = {
 493                 .vma = vma,
 494                 .addr1 = start,
 495                 .addr2 = end,
 496         };
 497         on_each_cpu(ipi_flush_tlb_range, &fd, 1);
 498 }
 499
 500 static void ipi_flush_tlb_kernel_range(void *arg)
 501 {
 502         struct flush_data *fd = arg;
 503         local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
 504 }
 505
 506 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 507 {
 508         struct flush_data fd = {
 509                 .addr1 = start,
 510                 .addr2 = end,
 511         };
 512         on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1);
 513 }
 514
 515 /* Cache flush functions */
 516
 517 static void ipi_flush_cache_all(void *arg)
 518 {
 519         local_flush_cache_all();
 520 }
 521
 522 void flush_cache_all(void)
 523 {
 524         on_each_cpu(ipi_flush_cache_all, NULL, 1);
 525 }
 526
 527 static void ipi_flush_cache_page(void *arg)
 528 {
 529         struct flush_data *fd = arg;
 530         local_flush_cache_page(fd->vma, fd->addr1, fd->addr2);
 531 }
 532
 533 void flush_cache_page(struct vm_area_struct *vma,
 534                      unsigned long address, unsigned long pfn)
 535 {
 536         struct flush_data fd = {
 537                 .vma = vma,
 538                 .addr1 = address,
 539                 .addr2 = pfn,
 540         };
 541         on_each_cpu(ipi_flush_cache_page, &fd, 1);
 542 }
 543
 544 static void ipi_flush_cache_range(void *arg)
 545 {
 546         struct flush_data *fd = arg;
 547         local_flush_cache_range(fd->vma, fd->addr1, fd->addr2);
 548 }
 549
 550 void flush_cache_range(struct vm_area_struct *vma,
 551                      unsigned long start, unsigned long end)
 552 {
 553         struct flush_data fd = {
 554                 .vma = vma,
 555                 .addr1 = start,
 556                 .addr2 = end,
 557         };
 558         on_each_cpu(ipi_flush_cache_range, &fd, 1);
 559 }
 560
 561 static void ipi_flush_icache_range(void *arg)
 562 {
 563         struct flush_data *fd = arg;
 564         local_flush_icache_range(fd->addr1, fd->addr2);
 565 }
 566
 567 void flush_icache_range(unsigned long start, unsigned long end)
 568 {
 569         struct flush_data fd = {
 570                 .addr1 = start,
 571                 .addr2 = end,
 572         };
 573         on_each_cpu(ipi_flush_icache_range, &fd, 1);
 574 }
 575 EXPORT_SYMBOL(flush_icache_range);
 576
 577 /* ------------------------------------------------------------------------- */
 578
 579 static void ipi_invalidate_dcache_range(void *arg)
 580 {
 581         struct flush_data *fd = arg;
 582         __invalidate_dcache_range(fd->addr1, fd->addr2);
 583 }
 584
 585 static void system_invalidate_dcache_range(unsigned long start,
 586                 unsigned long size)
 587 {
 588         struct flush_data fd = {
 589                 .addr1 = start,
 590                 .addr2 = size,
 591         };
 592         on_each_cpu(ipi_invalidate_dcache_range, &fd, 1);
 593 }
 594
 595 static void ipi_flush_invalidate_dcache_range(void *arg)
 596 {
 597         struct flush_data *fd = arg;
 598         __flush_invalidate_dcache_range(fd->addr1, fd->addr2);
 599 }
 600
 601 static void system_flush_invalidate_dcache_range(unsigned long start,
 602                 unsigned long size)
 603 {
 604         struct flush_data fd = {
 605                 .addr1 = start,
 606                 .addr2 = size,
 607         };
 608         on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1);
 609 }