arch/sparc64/kernel/smp.c

   1 /* smp.c: Sparc64 SMP support.
   2  *
   3  * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
   4  */
   5
   6 #include <linux/module.h>
   7 #include <linux/kernel.h>
   8 #include <linux/sched.h>
   9 #include <linux/mm.h>
  10 #include <linux/pagemap.h>
  11 #include <linux/threads.h>
  12 #include <linux/smp.h>
  13 #include <linux/smp_lock.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/kernel_stat.h>
  16 #include <linux/delay.h>
  17 #include <linux/init.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/fs.h>
  20 #include <linux/seq_file.h>
  21 #include <linux/cache.h>
  22 #include <linux/jiffies.h>
  23 #include <linux/profile.h>
  24 #include <linux/bootmem.h>
  25
  26 #include <asm/head.h>
  27 #include <asm/ptrace.h>
  28 #include <asm/atomic.h>
  29 #include <asm/tlbflush.h>
  30 #include <asm/mmu_context.h>
  31 #include <asm/cpudata.h>
  32
  33 #include <asm/irq.h>
  34 #include <asm/page.h>
  35 #include <asm/pgtable.h>
  36 #include <asm/oplib.h>
  37 #include <asm/uaccess.h>
  38 #include <asm/timer.h>
  39 #include <asm/starfire.h>
  40 #include <asm/tlb.h>
  41
  42 extern int linux_num_cpus;
  43 extern void calibrate_delay(void);
  44
  45 /* Please don't make this stuff initdata!!!  --DaveM */
  46 static unsigned char boot_cpu_id;
  47
  48 cpumask_t cpu_online_map = CPU_MASK_NONE;
  49 cpumask_t phys_cpu_present_map = CPU_MASK_NONE;
  50 static cpumask_t smp_commenced_mask;
  51 static cpumask_t cpu_callout_map;
  52
  53 void smp_info(struct seq_file *m)
  54 {
  55         int i;
  56
  57         seq_printf(m, "State:\n");
  58         for (i = 0; i < NR_CPUS; i++) {
  59                 if (cpu_online(i))
  60                         seq_printf(m,
  61                                    "CPU%d:\t\tonline\n", i);
  62         }
  63 }
  64
  65 void smp_bogo(struct seq_file *m)
  66 {
  67         int i;
  68
  69         for (i = 0; i < NR_CPUS; i++)
  70                 if (cpu_online(i))
  71                         seq_printf(m,
  72                                    "Cpu%dBogo\t: %lu.%02lu\n"
  73                                    "Cpu%dClkTck\t: %016lx\n",
  74                                    i, cpu_data(i).udelay_val / (500000/HZ),
  75                                    (cpu_data(i).udelay_val / (5000/HZ)) % 100,
  76                                    i, cpu_data(i).clock_tick);
  77 }
  78
  79 void __init smp_store_cpu_info(int id)
  80 {
  81         int cpu_node;
  82
  83         /* multiplier and counter set by
  84            smp_setup_percpu_timer()  */
  85         cpu_data(id).udelay_val                 = loops_per_jiffy;
  86
  87         cpu_find_by_mid(id, &cpu_node);
  88         cpu_data(id).clock_tick = prom_getintdefault(cpu_node,
  89                                                      "clock-frequency", 0);
  90
  91         cpu_data(id).pgcache_size               = 0;
  92         cpu_data(id).pte_cache[0]               = NULL;
  93         cpu_data(id).pte_cache[1]               = NULL;
  94         cpu_data(id).pgd_cache                  = NULL;
  95         cpu_data(id).idle_volume                = 1;
  96 }
  97
  98 static void smp_setup_percpu_timer(void);
  99
 100 static volatile unsigned long callin_flag = 0;
 101
 102 extern void inherit_locked_prom_mappings(int save_p);
 103
 104 static inline void cpu_setup_percpu_base(unsigned long cpu_id)
 105 {
 106         __asm__ __volatile__("mov       %0, %%g5\n\t"
 107                              "stxa      %0, [%1] %2\n\t"
 108                              "membar    #Sync"
 109                              : /* no outputs */
 110                              : "r" (__per_cpu_offset(cpu_id)),
 111                                "r" (TSB_REG), "i" (ASI_IMMU));
 112 }
 113
 114 void __init smp_callin(void)
 115 {
 116         int cpuid = hard_smp_processor_id();
 117
 118         inherit_locked_prom_mappings(0);
 119
 120         __flush_tlb_all();
 121
 122         cpu_setup_percpu_base(cpuid);
 123
 124         smp_setup_percpu_timer();
 125
 126         local_irq_enable();
 127
 128         calibrate_delay();
 129         smp_store_cpu_info(cpuid);
 130         callin_flag = 1;
 131         __asm__ __volatile__("membar #Sync\n\t"
 132                              "flush  %%g6" : : : "memory");
 133
 134         /* Clear this or we will die instantly when we
 135          * schedule back to this idler...
 136          */
 137         clear_thread_flag(TIF_NEWCHILD);
 138
 139         /* Attach to the address space of init_task. */
 140         atomic_inc(&init_mm.mm_count);
 141         current->active_mm = &init_mm;
 142
 143         while (!cpu_isset(cpuid, smp_commenced_mask))
 144                 membar("#LoadLoad");
 145
 146         cpu_set(cpuid, cpu_online_map);
 147 }
 148
 149 void cpu_panic(void)
 150 {
 151         printk("CPU[%d]: Returns from cpu_idle!\n", smp_processor_id());
 152         panic("SMP bolixed\n");
 153 }
 154
 155 static unsigned long current_tick_offset;
 156
 157 /* This tick register synchronization scheme is taken entirely from
 158  * the ia64 port, see arch/ia64/kernel/smpboot.c for details and credit.
 159  *
 160  * The only change I've made is to rework it so that the master
 161  * initiates the synchonization instead of the slave. -DaveM
 162  */
 163
 164 #define MASTER  0
 165 #define SLAVE   (SMP_CACHE_BYTES/sizeof(unsigned long))
 166
 167 #define NUM_ROUNDS      64      /* magic value */
 168 #define NUM_ITERS       5       /* likewise */
 169
 170 static DEFINE_SPINLOCK(itc_sync_lock);
 171 static unsigned long go[SLAVE + 1];
 172
 173 #define DEBUG_TICK_SYNC 0
 174
 175 static inline long get_delta (long *rt, long *master)
 176 {
 177         unsigned long best_t0 = 0, best_t1 = ~0UL, best_tm = 0;
 178         unsigned long tcenter, t0, t1, tm;
 179         unsigned long i;
 180
 181         for (i = 0; i < NUM_ITERS; i++) {
 182                 t0 = tick_ops->get_tick();
 183                 go[MASTER] = 1;
 184                 membar("#StoreLoad");
 185                 while (!(tm = go[SLAVE]))
 186                         membar("#LoadLoad");
 187                 go[SLAVE] = 0;
 188                 membar("#StoreStore");
 189                 t1 = tick_ops->get_tick();
 190
 191                 if (t1 - t0 < best_t1 - best_t0)
 192                         best_t0 = t0, best_t1 = t1, best_tm = tm;
 193         }
 194
 195         *rt = best_t1 - best_t0;
 196         *master = best_tm - best_t0;
 197
 198         /* average best_t0 and best_t1 without overflow: */
 199         tcenter = (best_t0/2 + best_t1/2);
 200         if (best_t0 % 2 + best_t1 % 2 == 2)
 201                 tcenter++;
 202         return tcenter - best_tm;
 203 }
 204
 205 void smp_synchronize_tick_client(void)
 206 {
 207         long i, delta, adj, adjust_latency = 0, done = 0;
 208         unsigned long flags, rt, master_time_stamp, bound;
 209 #if DEBUG_TICK_SYNC
 210         struct {
 211                 long rt;        /* roundtrip time */
 212                 long master;    /* master's timestamp */
 213                 long diff;      /* difference between midpoint and master's timestamp */
 214                 long lat;       /* estimate of itc adjustment latency */
 215         } t[NUM_ROUNDS];
 216 #endif
 217
 218         go[MASTER] = 1;
 219
 220         while (go[MASTER])
 221                 membar("#LoadLoad");
 222
 223         local_irq_save(flags);
 224         {
 225                 for (i = 0; i < NUM_ROUNDS; i++) {
 226                         delta = get_delta(&rt, &master_time_stamp);
 227                         if (delta == 0) {
 228                                 done = 1;       /* let's lock on to this... */
 229                                 bound = rt;
 230                         }
 231
 232                         if (!done) {
 233                                 if (i > 0) {
 234                                         adjust_latency += -delta;
 235                                         adj = -delta + adjust_latency/4;
 236                                 } else
 237                                         adj = -delta;
 238
 239                                 tick_ops->add_tick(adj, current_tick_offset);
 240                         }
 241 #if DEBUG_TICK_SYNC
 242                         t[i].rt = rt;
 243                         t[i].master = master_time_stamp;
 244                         t[i].diff = delta;
 245                         t[i].lat = adjust_latency/4;
 246 #endif
 247                 }
 248         }
 249         local_irq_restore(flags);
 250
 251 #if DEBUG_TICK_SYNC
 252         for (i = 0; i < NUM_ROUNDS; i++)
 253                 printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
 254                        t[i].rt, t[i].master, t[i].diff, t[i].lat);
 255 #endif
 256
 257         printk(KERN_INFO "CPU %d: synchronized TICK with master CPU (last diff %ld cycles,"
 258                "maxerr %lu cycles)\n", smp_processor_id(), delta, rt);
 259 }
 260
 261 static void smp_start_sync_tick_client(int cpu);
 262
 263 static void smp_synchronize_one_tick(int cpu)
 264 {
 265         unsigned long flags, i;
 266
 267         go[MASTER] = 0;
 268
 269         smp_start_sync_tick_client(cpu);
 270
 271         /* wait for client to be ready */
 272         while (!go[MASTER])
 273                 membar("#LoadLoad");
 274
 275         /* now let the client proceed into his loop */
 276         go[MASTER] = 0;
 277         membar("#StoreLoad");
 278
 279         spin_lock_irqsave(&itc_sync_lock, flags);
 280         {
 281                 for (i = 0; i < NUM_ROUNDS*NUM_ITERS; i++) {
 282                         while (!go[MASTER])
 283                                 membar("#LoadLoad");
 284                         go[MASTER] = 0;
 285                         membar("#StoreStore");
 286                         go[SLAVE] = tick_ops->get_tick();
 287                         membar("#StoreLoad");
 288                 }
 289         }
 290         spin_unlock_irqrestore(&itc_sync_lock, flags);
 291 }
 292
 293 extern unsigned long sparc64_cpu_startup;
 294
 295 /* The OBP cpu startup callback truncates the 3rd arg cookie to
 296  * 32-bits (I think) so to be safe we have it read the pointer
 297  * contained here so we work on >4GB machines. -DaveM
 298  */
 299 static struct thread_info *cpu_new_thread = NULL;
 300
 301 static int __devinit smp_boot_one_cpu(unsigned int cpu)
 302 {
 303         unsigned long entry =
 304                 (unsigned long)(&sparc64_cpu_startup);
 305         unsigned long cookie =
 306                 (unsigned long)(&cpu_new_thread);
 307         struct task_struct *p;
 308         int timeout, ret, cpu_node;
 309
 310         p = fork_idle(cpu);
 311         callin_flag = 0;
 312         cpu_new_thread = p->thread_info;
 313         cpu_set(cpu, cpu_callout_map);
 314
 315         cpu_find_by_mid(cpu, &cpu_node);
 316         prom_startcpu(cpu_node, entry, cookie);
 317
 318         for (timeout = 0; timeout < 5000000; timeout++) {
 319                 if (callin_flag)
 320                         break;
 321                 udelay(100);
 322         }
 323         if (callin_flag) {
 324                 ret = 0;
 325         } else {
 326                 printk("Processor %d is stuck.\n", cpu);
 327                 cpu_clear(cpu, cpu_callout_map);
 328                 ret = -ENODEV;
 329         }
 330         cpu_new_thread = NULL;
 331
 332         return ret;
 333 }
 334
 335 static void spitfire_xcall_helper(u64 data0, u64 data1, u64 data2, u64 pstate, unsigned long cpu)
 336 {
 337         u64 result, target;
 338         int stuck, tmp;
 339
 340         if (this_is_starfire) {
 341                 /* map to real upaid */
 342                 cpu = (((cpu & 0x3c) << 1) |
 343                         ((cpu & 0x40) >> 4) |
 344                         (cpu & 0x3));
 345         }
 346
 347         target = (cpu << 14) | 0x70;
 348 again:
 349         /* Ok, this is the real Spitfire Errata #54.
 350          * One must read back from a UDB internal register
 351          * after writes to the UDB interrupt dispatch, but
 352          * before the membar Sync for that write.
 353          * So we use the high UDB control register (ASI 0x7f,
 354          * ADDR 0x20) for the dummy read. -DaveM
 355          */
 356         tmp = 0x40;
 357         __asm__ __volatile__(
 358         "wrpr   %1, %2, %%pstate\n\t"
 359         "stxa   %4, [%0] %3\n\t"
 360         "stxa   %5, [%0+%8] %3\n\t"
 361         "add    %0, %8, %0\n\t"
 362         "stxa   %6, [%0+%8] %3\n\t"
 363         "membar #Sync\n\t"
 364         "stxa   %%g0, [%7] %3\n\t"
 365         "membar #Sync\n\t"
 366         "mov    0x20, %%g1\n\t"
 367         "ldxa   [%%g1] 0x7f, %%g0\n\t"
 368         "membar #Sync"
 369         : "=r" (tmp)
 370         : "r" (pstate), "i" (PSTATE_IE), "i" (ASI_INTR_W),
 371           "r" (data0), "r" (data1), "r" (data2), "r" (target),
 372           "r" (0x10), "0" (tmp)
 373         : "g1");
 374
 375         /* NOTE: PSTATE_IE is still clear. */
 376         stuck = 100000;
 377         do {
 378                 __asm__ __volatile__("ldxa [%%g0] %1, %0"
 379                         : "=r" (result)
 380                         : "i" (ASI_INTR_DISPATCH_STAT));
 381                 if (result == 0) {
 382                         __asm__ __volatile__("wrpr %0, 0x0, %%pstate"
 383                                              : : "r" (pstate));
 384                         return;
 385                 }
 386                 stuck -= 1;
 387                 if (stuck == 0)
 388                         break;
 389         } while (result & 0x1);
 390         __asm__ __volatile__("wrpr %0, 0x0, %%pstate"
 391                              : : "r" (pstate));
 392         if (stuck == 0) {
 393                 printk("CPU[%d]: mondo stuckage result[%016lx]\n",
 394                        smp_processor_id(), result);
 395         } else {
 396                 udelay(2);
 397                 goto again;
 398         }
 399 }
 400
 401 static __inline__ void spitfire_xcall_deliver(u64 data0, u64 data1, u64 data2, cpumask_t mask)
 402 {
 403         u64 pstate;
 404         int i;
 405
 406         __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate));
 407         for_each_cpu_mask(i, mask)
 408                 spitfire_xcall_helper(data0, data1, data2, pstate, i);
 409 }
 410
 411 /* Cheetah now allows to send the whole 64-bytes of data in the interrupt
 412  * packet, but we have no use for that.  However we do take advantage of
 413  * the new pipelining feature (ie. dispatch to multiple cpus simultaneously).
 414  */
 415 static void cheetah_xcall_deliver(u64 data0, u64 data1, u64 data2, cpumask_t mask)
 416 {
 417         u64 pstate, ver;
 418         int nack_busy_id, is_jalapeno;
 419
 420         if (cpus_empty(mask))
 421                 return;
 422
 423         /* Unfortunately, someone at Sun had the brilliant idea to make the
 424          * busy/nack fields hard-coded by ITID number for this Ultra-III
 425          * derivative processor.
 426          */
 427         __asm__ ("rdpr %%ver, %0" : "=r" (ver));
 428         is_jalapeno = ((ver >> 32) == 0x003e0016);
 429
 430         __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate));
 431
 432 retry:
 433         __asm__ __volatile__("wrpr %0, %1, %%pstate\n\t"
 434                              : : "r" (pstate), "i" (PSTATE_IE));
 435
 436         /* Setup the dispatch data registers. */
 437         __asm__ __volatile__("stxa      %0, [%3] %6\n\t"
 438                              "stxa      %1, [%4] %6\n\t"
 439                              "stxa      %2, [%5] %6\n\t"
 440                              "membar    #Sync\n\t"
 441                              : /* no outputs */
 442                              : "r" (data0), "r" (data1), "r" (data2),
 443                                "r" (0x40), "r" (0x50), "r" (0x60),
 444                                "i" (ASI_INTR_W));
 445
 446         nack_busy_id = 0;
 447         {
 448                 int i;
 449
 450                 for_each_cpu_mask(i, mask) {
 451                         u64 target = (i << 14) | 0x70;
 452
 453                         if (!is_jalapeno)
 454                                 target |= (nack_busy_id << 24);
 455                         __asm__ __volatile__(
 456                                 "stxa   %%g0, [%0] %1\n\t"
 457                                 "membar #Sync\n\t"
 458                                 : /* no outputs */
 459                                 : "r" (target), "i" (ASI_INTR_W));
 460                         nack_busy_id++;
 461                 }
 462         }
 463
 464         /* Now, poll for completion. */
 465         {
 466                 u64 dispatch_stat;
 467                 long stuck;
 468
 469                 stuck = 100000 * nack_busy_id;
 470                 do {
 471                         __asm__ __volatile__("ldxa      [%%g0] %1, %0"
 472                                              : "=r" (dispatch_stat)
 473                                              : "i" (ASI_INTR_DISPATCH_STAT));
 474                         if (dispatch_stat == 0UL) {
 475                                 __asm__ __volatile__("wrpr %0, 0x0, %%pstate"
 476                                                      : : "r" (pstate));
 477                                 return;
 478                         }
 479                         if (!--stuck)
 480                                 break;
 481                 } while (dispatch_stat & 0x5555555555555555UL);
 482
 483                 __asm__ __volatile__("wrpr %0, 0x0, %%pstate"
 484                                      : : "r" (pstate));
 485
 486                 if ((dispatch_stat & ~(0x5555555555555555UL)) == 0) {
 487                         /* Busy bits will not clear, continue instead
 488                          * of freezing up on this cpu.
 489                          */
 490                         printk("CPU[%d]: mondo stuckage result[%016lx]\n",
 491                                smp_processor_id(), dispatch_stat);
 492                 } else {
 493                         int i, this_busy_nack = 0;
 494
 495                         /* Delay some random time with interrupts enabled
 496                          * to prevent deadlock.
 497                          */
 498                         udelay(2 * nack_busy_id);
 499
 500                         /* Clear out the mask bits for cpus which did not
 501                          * NACK us.
 502                          */
 503                         for_each_cpu_mask(i, mask) {
 504                                 u64 check_mask;
 505
 506                                 if (is_jalapeno)
 507                                         check_mask = (0x2UL << (2*i));
 508                                 else
 509                                         check_mask = (0x2UL <<
 510                                                       this_busy_nack);
 511                                 if ((dispatch_stat & check_mask) == 0)
 512                                         cpu_clear(i, mask);
 513                                 this_busy_nack += 2;
 514                         }
 515
 516                         goto retry;
 517                 }
 518         }
 519 }
 520
 521 /* Send cross call to all processors mentioned in MASK
 522  * except self.
 523  */
 524 static void smp_cross_call_masked(unsigned long *func, u32 ctx, u64 data1, u64 data2, cpumask_t mask)
 525 {
 526         u64 data0 = (((u64)ctx)<<32 | (((u64)func) & 0xffffffff));
 527         int this_cpu = get_cpu();
 528
 529         cpus_and(mask, mask, cpu_online_map);
 530         cpu_clear(this_cpu, mask);
 531
 532         if (tlb_type == spitfire)
 533                 spitfire_xcall_deliver(data0, data1, data2, mask);
 534         else
 535                 cheetah_xcall_deliver(data0, data1, data2, mask);
 536         /* NOTE: Caller runs local copy on master. */
 537
 538         put_cpu();
 539 }
 540
 541 extern unsigned long xcall_sync_tick;
 542
 543 static void smp_start_sync_tick_client(int cpu)
 544 {
 545         cpumask_t mask = cpumask_of_cpu(cpu);
 546
 547         smp_cross_call_masked(&xcall_sync_tick,
 548                               0, 0, 0, mask);
 549 }
 550
 551 /* Send cross call to all processors except self. */
 552 #define smp_cross_call(func, ctx, data1, data2) \
 553         smp_cross_call_masked(func, ctx, data1, data2, cpu_online_map)
 554
 555 struct call_data_struct {
 556         void (*func) (void *info);
 557         void *info;
 558         atomic_t finished;
 559         int wait;
 560 };
 561
 562 static DEFINE_SPINLOCK(call_lock);
 563 static struct call_data_struct *call_data;
 564
 565 extern unsigned long xcall_call_function;
 566
 567 /*
 568  * You must not call this function with disabled interrupts or from a
 569  * hardware interrupt handler or from a bottom half handler.
 570  */
 571 int smp_call_function(void (*func)(void *info), void *info,
 572                       int nonatomic, int wait)
 573 {
 574         struct call_data_struct data;
 575         int cpus = num_online_cpus() - 1;
 576         long timeout;
 577
 578         if (!cpus)
 579                 return 0;
 580
 581         /* Can deadlock when called with interrupts disabled */
 582         WARN_ON(irqs_disabled());
 583
 584         data.func = func;
 585         data.info = info;
 586         atomic_set(&data.finished, 0);
 587         data.wait = wait;
 588
 589         spin_lock(&call_lock);
 590
 591         call_data = &data;
 592
 593         smp_cross_call(&xcall_call_function, 0, 0, 0);
 594
 595         /*
 596          * Wait for other cpus to complete function or at
 597          * least snap the call data.
 598          */
 599         timeout = 1000000;
 600         while (atomic_read(&data.finished) != cpus) {
 601                 if (--timeout <= 0)
 602                         goto out_timeout;
 603                 barrier();
 604                 udelay(1);
 605         }
 606
 607         spin_unlock(&call_lock);
 608
 609         return 0;
 610
 611 out_timeout:
 612         spin_unlock(&call_lock);
 613         printk("XCALL: Remote cpus not responding, ncpus=%ld finished=%ld\n",
 614                (long) num_online_cpus() - 1L,
 615                (long) atomic_read(&data.finished));
 616         return 0;
 617 }
 618
 619 void smp_call_function_client(int irq, struct pt_regs *regs)
 620 {
 621         void (*func) (void *info) = call_data->func;
 622         void *info = call_data->info;
 623
 624         clear_softint(1 << irq);
 625         if (call_data->wait) {
 626                 /* let initiator proceed only after completion */
 627                 func(info);
 628                 atomic_inc(&call_data->finished);
 629         } else {
 630                 /* let initiator proceed after getting data */
 631                 atomic_inc(&call_data->finished);
 632                 func(info);
 633         }
 634 }
 635
 636 extern unsigned long xcall_flush_tlb_mm;
 637 extern unsigned long xcall_flush_tlb_pending;
 638 extern unsigned long xcall_flush_tlb_kernel_range;
 639 extern unsigned long xcall_flush_tlb_all_spitfire;
 640 extern unsigned long xcall_flush_tlb_all_cheetah;
 641 extern unsigned long xcall_report_regs;
 642 extern unsigned long xcall_receive_signal;
 643
 644 #ifdef DCACHE_ALIASING_POSSIBLE
 645 extern unsigned long xcall_flush_dcache_page_cheetah;
 646 #endif
 647 extern unsigned long xcall_flush_dcache_page_spitfire;
 648
 649 #ifdef CONFIG_DEBUG_DCFLUSH
 650 extern atomic_t dcpage_flushes;
 651 extern atomic_t dcpage_flushes_xcall;
 652 #endif
 653
 654 static __inline__ void __local_flush_dcache_page(struct page *page)
 655 {
 656 #ifdef DCACHE_ALIASING_POSSIBLE
 657         __flush_dcache_page(page_address(page),
 658                             ((tlb_type == spitfire) &&
 659                              page_mapping(page) != NULL));
 660 #else
 661         if (page_mapping(page) != NULL &&
 662             tlb_type == spitfire)
 663                 __flush_icache_page(__pa(page_address(page)));
 664 #endif
 665 }
 666
 667 void smp_flush_dcache_page_impl(struct page *page, int cpu)
 668 {
 669         cpumask_t mask = cpumask_of_cpu(cpu);
 670         int this_cpu = get_cpu();
 671
 672 #ifdef CONFIG_DEBUG_DCFLUSH
 673         atomic_inc(&dcpage_flushes);
 674 #endif
 675         if (cpu == this_cpu) {
 676                 __local_flush_dcache_page(page);
 677         } else if (cpu_online(cpu)) {
 678                 void *pg_addr = page_address(page);
 679                 u64 data0;
 680
 681                 if (tlb_type == spitfire) {
 682                         data0 =
 683                                 ((u64)&xcall_flush_dcache_page_spitfire);
 684                         if (page_mapping(page) != NULL)
 685                                 data0 |= ((u64)1 << 32);
 686                         spitfire_xcall_deliver(data0,
 687                                                __pa(pg_addr),
 688                                                (u64) pg_addr,
 689                                                mask);
 690                 } else {
 691 #ifdef DCACHE_ALIASING_POSSIBLE
 692                         data0 =
 693                                 ((u64)&xcall_flush_dcache_page_cheetah);
 694                         cheetah_xcall_deliver(data0,
 695                                               __pa(pg_addr),
 696                                               0, mask);
 697 #endif
 698                 }
 699 #ifdef CONFIG_DEBUG_DCFLUSH
 700                 atomic_inc(&dcpage_flushes_xcall);
 701 #endif
 702         }
 703
 704         put_cpu();
 705 }
 706
 707 void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
 708 {
 709         void *pg_addr = page_address(page);
 710         cpumask_t mask = cpu_online_map;
 711         u64 data0;
 712         int this_cpu = get_cpu();
 713
 714         cpu_clear(this_cpu, mask);
 715
 716 #ifdef CONFIG_DEBUG_DCFLUSH
 717         atomic_inc(&dcpage_flushes);
 718 #endif
 719         if (cpus_empty(mask))
 720                 goto flush_self;
 721         if (tlb_type == spitfire) {
 722                 data0 = ((u64)&xcall_flush_dcache_page_spitfire);
 723                 if (page_mapping(page) != NULL)
 724                         data0 |= ((u64)1 << 32);
 725                 spitfire_xcall_deliver(data0,
 726                                        __pa(pg_addr),
 727                                        (u64) pg_addr,
 728                                        mask);
 729         } else {
 730 #ifdef DCACHE_ALIASING_POSSIBLE
 731                 data0 = ((u64)&xcall_flush_dcache_page_cheetah);
 732                 cheetah_xcall_deliver(data0,
 733                                       __pa(pg_addr),
 734                                       0, mask);
 735 #endif
 736         }
 737 #ifdef CONFIG_DEBUG_DCFLUSH
 738         atomic_inc(&dcpage_flushes_xcall);
 739 #endif
 740  flush_self:
 741         __local_flush_dcache_page(page);
 742
 743         put_cpu();
 744 }
 745
 746 void smp_receive_signal(int cpu)
 747 {
 748         cpumask_t mask = cpumask_of_cpu(cpu);
 749
 750         if (cpu_online(cpu)) {
 751                 u64 data0 = (((u64)&xcall_receive_signal) & 0xffffffff);
 752
 753                 if (tlb_type == spitfire)
 754                         spitfire_xcall_deliver(data0, 0, 0, mask);
 755                 else
 756                         cheetah_xcall_deliver(data0, 0, 0, mask);
 757         }
 758 }
 759
 760 void smp_receive_signal_client(int irq, struct pt_regs *regs)
 761 {
 762         /* Just return, rtrap takes care of the rest. */
 763         clear_softint(1 << irq);
 764 }
 765
 766 void smp_report_regs(void)
 767 {
 768         smp_cross_call(&xcall_report_regs, 0, 0, 0);
 769 }
 770
 771 void smp_flush_tlb_all(void)
 772 {
 773         if (tlb_type == spitfire)
 774                 smp_cross_call(&xcall_flush_tlb_all_spitfire, 0, 0, 0);
 775         else
 776                 smp_cross_call(&xcall_flush_tlb_all_cheetah, 0, 0, 0);
 777         __flush_tlb_all();
 778 }
 779
 780 /* We know that the window frames of the user have been flushed
 781  * to the stack before we get here because all callers of us
 782  * are flush_tlb_*() routines, and these run after flush_cache_*()
 783  * which performs the flushw.
 784  *
 785  * The SMP TLB coherency scheme we use works as follows:
 786  *
 787  * 1) mm->cpu_vm_mask is a bit mask of which cpus an address
 788  *    space has (potentially) executed on, this is the heuristic
 789  *    we use to avoid doing cross calls.
 790  *
 791  *    Also, for flushing from kswapd and also for clones, we
 792  *    use cpu_vm_mask as the list of cpus to make run the TLB.
 793  *
 794  * 2) TLB context numbers are shared globally across all processors
 795  *    in the system, this allows us to play several games to avoid
 796  *    cross calls.
 797  *
 798  *    One invariant is that when a cpu switches to a process, and
 799  *    that processes tsk->active_mm->cpu_vm_mask does not have the
 800  *    current cpu's bit set, that tlb context is flushed locally.
 801  *
 802  *    If the address space is non-shared (ie. mm->count == 1) we avoid
 803  *    cross calls when we want to flush the currently running process's
 804  *    tlb state.  This is done by clearing all cpu bits except the current
 805  *    processor's in current->active_mm->cpu_vm_mask and performing the
 806  *    flush locally only.  This will force any subsequent cpus which run
 807  *    this task to flush the context from the local tlb if the process
 808  *    migrates to another cpu (again).
 809  *
 810  * 3) For shared address spaces (threads) and swapping we bite the
 811  *    bullet for most cases and perform the cross call (but only to
 812  *    the cpus listed in cpu_vm_mask).
 813  *
 814  *    The performance gain from "optimizing" away the cross call for threads is
 815  *    questionable (in theory the big win for threads is the massive sharing of
 816  *    address space state across processors).
 817  */
 818 void smp_flush_tlb_mm(struct mm_struct *mm)
 819 {
 820         /*
 821          * This code is called from two places, dup_mmap and exit_mmap. In the
 822          * former case, we really need a flush. In the later case, the callers
 823          * are single threaded exec_mmap (really need a flush), multithreaded
 824          * exec_mmap case (do not need to flush, since the caller gets a new
 825          * context via activate_mm), and all other callers of mmput() whence
 826          * the flush can be optimized since the associated threads are dead and
 827          * the mm is being torn down (__exit_mm and other mmput callers) or the
 828          * owning thread is dissociating itself from the mm. The
 829          * (atomic_read(&mm->mm_users) == 0) check ensures real work is done
 830          * for single thread exec and dup_mmap cases. An alternate check might
 831          * have been (current->mm != mm).
 832          *                                              Kanoj Sarcar
 833          */
 834         if (atomic_read(&mm->mm_users) == 0)
 835                 return;
 836
 837         {
 838                 u32 ctx = CTX_HWBITS(mm->context);
 839                 int cpu = get_cpu();
 840
 841                 if (atomic_read(&mm->mm_users) == 1) {
 842                         mm->cpu_vm_mask = cpumask_of_cpu(cpu);
 843                         goto local_flush_and_out;
 844                 }
 845
 846                 smp_cross_call_masked(&xcall_flush_tlb_mm,
 847                                       ctx, 0, 0,
 848                                       mm->cpu_vm_mask);
 849
 850         local_flush_and_out:
 851                 __flush_tlb_mm(ctx, SECONDARY_CONTEXT);
 852
 853                 put_cpu();
 854         }
 855 }
 856
 857 void smp_flush_tlb_pending(struct mm_struct *mm, unsigned long nr, unsigned long *vaddrs)
 858 {
 859         u32 ctx = CTX_HWBITS(mm->context);
 860         int cpu = get_cpu();
 861
 862         if (mm == current->active_mm && atomic_read(&mm->mm_users) == 1) {
 863                 mm->cpu_vm_mask = cpumask_of_cpu(cpu);
 864                 goto local_flush_and_out;
 865         } else {
 866                 /* This optimization is not valid.  Normally
 867                  * we will be holding the page_table_lock, but
 868                  * there is an exception which is copy_page_range()
 869                  * when forking.  The lock is held during the individual
 870                  * page table updates in the parent, but not at the
 871                  * top level, which is where we are invoked.
 872                  */
 873                 if (0) {
 874                         cpumask_t this_cpu_mask = cpumask_of_cpu(cpu);
 875
 876                         /* By virtue of running under the mm->page_table_lock,
 877                          * and mmu_context.h:switch_mm doing the same, the
 878                          * following operation is safe.
 879                          */
 880                         if (cpus_equal(mm->cpu_vm_mask, this_cpu_mask))
 881                                 goto local_flush_and_out;
 882                 }
 883         }
 884
 885         smp_cross_call_masked(&xcall_flush_tlb_pending,
 886                               ctx, nr, (unsigned long) vaddrs,
 887                               mm->cpu_vm_mask);
 888
 889 local_flush_and_out:
 890         __flush_tlb_pending(ctx, nr, vaddrs);
 891
 892         put_cpu();
 893 }
 894
 895 void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 896 {
 897         start &= PAGE_MASK;
 898         end    = PAGE_ALIGN(end);
 899         if (start != end) {
 900                 smp_cross_call(&xcall_flush_tlb_kernel_range,
 901                                0, start, end);
 902
 903                 __flush_tlb_kernel_range(start, end);
 904         }
 905 }
 906
 907 /* CPU capture. */
 908 /* #define CAPTURE_DEBUG */
 909 extern unsigned long xcall_capture;
 910
 911 static atomic_t smp_capture_depth = ATOMIC_INIT(0);
 912 static atomic_t smp_capture_registry = ATOMIC_INIT(0);
 913 static unsigned long penguins_are_doing_time;
 914
 915 void smp_capture(void)
 916 {
 917         int result = atomic_add_ret(1, &smp_capture_depth);
 918
 919         if (result == 1) {
 920                 int ncpus = num_online_cpus();
 921
 922 #ifdef CAPTURE_DEBUG
 923                 printk("CPU[%d]: Sending penguins to jail...",
 924                        smp_processor_id());
 925 #endif
 926                 penguins_are_doing_time = 1;
 927                 membar("#StoreStore | #LoadStore");
 928                 atomic_inc(&smp_capture_registry);
 929                 smp_cross_call(&xcall_capture, 0, 0, 0);
 930                 while (atomic_read(&smp_capture_registry) != ncpus)
 931                         membar("#LoadLoad");
 932 #ifdef CAPTURE_DEBUG
 933                 printk("done\n");
 934 #endif
 935         }
 936 }
 937
 938 void smp_release(void)
 939 {
 940         if (atomic_dec_and_test(&smp_capture_depth)) {
 941 #ifdef CAPTURE_DEBUG
 942                 printk("CPU[%d]: Giving pardon to "
 943                        "imprisoned penguins\n",
 944                        smp_processor_id());
 945 #endif
 946                 penguins_are_doing_time = 0;
 947                 membar("#StoreStore | #StoreLoad");
 948                 atomic_dec(&smp_capture_registry);
 949         }
 950 }
 951
 952 /* Imprisoned penguins run with %pil == 15, but PSTATE_IE set, so they
 953  * can service tlb flush xcalls...
 954  */
 955 extern void prom_world(int);
 956 extern void save_alternate_globals(unsigned long *);
 957 extern void restore_alternate_globals(unsigned long *);
 958 void smp_penguin_jailcell(int irq, struct pt_regs *regs)
 959 {
 960         unsigned long global_save[24];
 961
 962         clear_softint(1 << irq);
 963
 964         preempt_disable();
 965
 966         __asm__ __volatile__("flushw");
 967         save_alternate_globals(global_save);
 968         prom_world(1);
 969         atomic_inc(&smp_capture_registry);
 970         membar("#StoreLoad | #StoreStore");
 971         while (penguins_are_doing_time)
 972                 membar("#LoadLoad");
 973         restore_alternate_globals(global_save);
 974         atomic_dec(&smp_capture_registry);
 975         prom_world(0);
 976
 977         preempt_enable();
 978 }
 979
 980 extern unsigned long xcall_promstop;
 981
 982 void smp_promstop_others(void)
 983 {
 984         smp_cross_call(&xcall_promstop, 0, 0, 0);
 985 }
 986
 987 #define prof_multiplier(__cpu)          cpu_data(__cpu).multiplier
 988 #define prof_counter(__cpu)             cpu_data(__cpu).counter
 989
 990 void smp_percpu_timer_interrupt(struct pt_regs *regs)
 991 {
 992         unsigned long compare, tick, pstate;
 993         int cpu = smp_processor_id();
 994         int user = user_mode(regs);
 995
 996         /*
 997          * Check for level 14 softint.
 998          */
 999         {
1000                 unsigned long tick_mask = tick_ops->softint_mask;
1001
1002                 if (!(get_softint() & tick_mask)) {
1003                         extern void handler_irq(int, struct pt_regs *);
1004
1005                         handler_irq(14, regs);
1006                         return;
1007                 }
1008                 clear_softint(tick_mask);
1009         }
1010
1011         do {
1012                 profile_tick(CPU_PROFILING, regs);
1013                 if (!--prof_counter(cpu)) {
1014                         irq_enter();
1015
1016                         if (cpu == boot_cpu_id) {
1017                                 kstat_this_cpu.irqs[0]++;
1018                                 timer_tick_interrupt(regs);
1019                         }
1020
1021                         update_process_times(user);
1022
1023                         irq_exit();
1024
1025                         prof_counter(cpu) = prof_multiplier(cpu);
1026                 }
1027
1028                 /* Guarantee that the following sequences execute
1029                  * uninterrupted.
1030                  */
1031                 __asm__ __volatile__("rdpr      %%pstate, %0\n\t"
1032                                      "wrpr      %0, %1, %%pstate"
1033                                      : "=r" (pstate)
1034                                      : "i" (PSTATE_IE));
1035
1036                 compare = tick_ops->add_compare(current_tick_offset);
1037                 tick = tick_ops->get_tick();
1038
1039                 /* Restore PSTATE_IE. */
1040                 __asm__ __volatile__("wrpr      %0, 0x0, %%pstate"
1041                                      : /* no outputs */
1042                                      : "r" (pstate));
1043         } while (time_after_eq(tick, compare));
1044 }
1045
1046 static void __init smp_setup_percpu_timer(void)
1047 {
1048         int cpu = smp_processor_id();
1049         unsigned long pstate;
1050
1051         prof_counter(cpu) = prof_multiplier(cpu) = 1;
1052
1053         /* Guarantee that the following sequences execute
1054          * uninterrupted.
1055          */
1056         __asm__ __volatile__("rdpr      %%pstate, %0\n\t"
1057                              "wrpr      %0, %1, %%pstate"
1058                              : "=r" (pstate)
1059                              : "i" (PSTATE_IE));
1060
1061         tick_ops->init_tick(current_tick_offset);
1062
1063         /* Restore PSTATE_IE. */
1064         __asm__ __volatile__("wrpr      %0, 0x0, %%pstate"
1065                              : /* no outputs */
1066                              : "r" (pstate));
1067 }
1068
1069 void __init smp_tick_init(void)
1070 {
1071         boot_cpu_id = hard_smp_processor_id();
1072         current_tick_offset = timer_tick_offset;
1073
1074         cpu_set(boot_cpu_id, cpu_online_map);
1075         prof_counter(boot_cpu_id) = prof_multiplier(boot_cpu_id) = 1;
1076 }
1077
1078 /* /proc/profile writes can call this, don't __init it please. */
1079 static DEFINE_SPINLOCK(prof_setup_lock);
1080
1081 int setup_profiling_timer(unsigned int multiplier)
1082 {
1083         unsigned long flags;
1084         int i;
1085
1086         if ((!multiplier) || (timer_tick_offset / multiplier) < 1000)
1087                 return -EINVAL;
1088
1089         spin_lock_irqsave(&prof_setup_lock, flags);
1090         for (i = 0; i < NR_CPUS; i++)
1091                 prof_multiplier(i) = multiplier;
1092         current_tick_offset = (timer_tick_offset / multiplier);
1093         spin_unlock_irqrestore(&prof_setup_lock, flags);
1094
1095         return 0;
1096 }
1097
1098 void __init smp_prepare_cpus(unsigned int max_cpus)
1099 {
1100         int instance, mid;
1101
1102         instance = 0;
1103         while (!cpu_find_by_instance(instance, NULL, &mid)) {
1104                 if (mid < max_cpus)
1105                         cpu_set(mid, phys_cpu_present_map);
1106                 instance++;
1107         }
1108
1109         if (num_possible_cpus() > max_cpus) {
1110                 instance = 0;
1111                 while (!cpu_find_by_instance(instance, NULL, &mid)) {
1112                         if (mid != boot_cpu_id) {
1113                                 cpu_clear(mid, phys_cpu_present_map);
1114                                 if (num_possible_cpus() <= max_cpus)
1115                                         break;
1116                         }
1117                         instance++;
1118                 }
1119         }
1120
1121         smp_store_cpu_info(boot_cpu_id);
1122 }
1123
1124 void __devinit smp_prepare_boot_cpu(void)
1125 {
1126         if (hard_smp_processor_id() >= NR_CPUS) {
1127                 prom_printf("Serious problem, boot cpu id >= NR_CPUS\n");
1128                 prom_halt();
1129         }
1130
1131         current_thread_info()->cpu = hard_smp_processor_id();
1132
1133         cpu_set(smp_processor_id(), cpu_online_map);
1134         cpu_set(smp_processor_id(), phys_cpu_present_map);
1135 }
1136
1137 int __devinit __cpu_up(unsigned int cpu)
1138 {
1139         int ret = smp_boot_one_cpu(cpu);
1140
1141         if (!ret) {
1142                 cpu_set(cpu, smp_commenced_mask);
1143                 while (!cpu_isset(cpu, cpu_online_map))
1144                         mb();
1145                 if (!cpu_isset(cpu, cpu_online_map)) {
1146                         ret = -ENODEV;
1147                 } else {
1148                         smp_synchronize_one_tick(cpu);
1149                 }
1150         }
1151         return ret;
1152 }
1153
1154 void __init smp_cpus_done(unsigned int max_cpus)
1155 {
1156         unsigned long bogosum = 0;
1157         int i;
1158
1159         for (i = 0; i < NR_CPUS; i++) {
1160                 if (cpu_online(i))
1161                         bogosum += cpu_data(i).udelay_val;
1162         }
1163         printk("Total of %ld processors activated "
1164                "(%lu.%02lu BogoMIPS).\n",
1165                (long) num_online_cpus(),
1166                bogosum/(500000/HZ),
1167                (bogosum/(5000/HZ))%100);
1168 }
1169
1170 /* This needn't do anything as we do not sleep the cpu
1171  * inside of the idler task, so an interrupt is not needed
1172  * to get a clean fast response.
1173  *
1174  * XXX Reverify this assumption... -DaveM
1175  *
1176  * Addendum: We do want it to do something for the signal
1177  *           delivery case, we detect that by just seeing
1178  *           if we are trying to send this to an idler or not.
1179  */
1180 void smp_send_reschedule(int cpu)
1181 {
1182         if (cpu_data(cpu).idle_volume == 0)
1183                 smp_receive_signal(cpu);
1184 }
1185
1186 /* This is a nop because we capture all other cpus
1187  * anyways when making the PROM active.
1188  */
1189 void smp_send_stop(void)
1190 {
1191 }
1192
1193 unsigned long __per_cpu_base;
1194 unsigned long __per_cpu_shift;
1195
1196 EXPORT_SYMBOL(__per_cpu_base);
1197 EXPORT_SYMBOL(__per_cpu_shift);
1198
1199 void __init setup_per_cpu_areas(void)
1200 {
1201         unsigned long goal, size, i;
1202         char *ptr;
1203         /* Created by linker magic */
1204         extern char __per_cpu_start[], __per_cpu_end[];
1205
1206         /* Copy section for each CPU (we discard the original) */
1207         goal = ALIGN(__per_cpu_end - __per_cpu_start, PAGE_SIZE);
1208
1209 #ifdef CONFIG_MODULES
1210         if (goal < PERCPU_ENOUGH_ROOM)
1211                 goal = PERCPU_ENOUGH_ROOM;
1212 #endif
1213         __per_cpu_shift = 0;
1214         for (size = 1UL; size < goal; size <<= 1UL)
1215                 __per_cpu_shift++;
1216
1217         /* Make sure the resulting __per_cpu_base value
1218          * will fit in the 43-bit sign extended IMMU
1219          * TSB register.
1220          */
1221         ptr = __alloc_bootmem(size * NR_CPUS, PAGE_SIZE,
1222                               (unsigned long) __per_cpu_start);
1223
1224         __per_cpu_base = ptr - __per_cpu_start;
1225
1226         if ((__per_cpu_shift < PAGE_SHIFT) ||
1227             (__per_cpu_base & ~PAGE_MASK) ||
1228             (__per_cpu_base != (((long) __per_cpu_base << 20) >> 20))) {
1229                 prom_printf("PER_CPU: Invalid layout, "
1230                             "ptr[%p] shift[%lx] base[%lx]\n",
1231                             ptr, __per_cpu_shift, __per_cpu_base);
1232                 prom_halt();
1233         }
1234
1235         for (i = 0; i < NR_CPUS; i++, ptr += size)
1236                 memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
1237
1238         /* Finally, load in the boot cpu's base value.
1239          * We abuse the IMMU TSB register for trap handler
1240          * entry and exit loading of %g5.  That is why it
1241          * has to be page aligned.
1242          */
1243         cpu_setup_percpu_base(hard_smp_processor_id());
1244 }