kernel/fiasco/src/kern/thread.cpp

   1 INTERFACE:
   2
   3 #include "l4_types.h"
   4 #include "config.h"
   5 #include "continuation.h"
   6 #include "helping_lock.h"
   7 #include "kobject.h"
   8 #include "mem_layout.h"
   9 #include "member_offs.h"
  10 #include "receiver.h"
  11 #include "ref_obj.h"
  12 #include "sender.h"
  13 #include "space.h"              // Space_index
  14 #include "spin_lock.h"
  15 #include "thread_lock.h"
  16
  17 class Return_frame;
  18 class Syscall_frame;
  19 class Vcpu_state;
  20 class Irq_base;
  21
  22 typedef Context_ptr_base<Thread> Thread_ptr;
  23
  24
  25 /** A thread.  This class is the driver class for most kernel functionality.
  26  */
  27 class Thread :
  28   public Receiver,
  29   public Sender,
  30   public Kobject
  31 {
  32   MEMBER_OFFSET();
  33   FIASCO_DECLARE_KOBJ();
  34
  35   friend class Jdb;
  36   friend class Jdb_bt;
  37   friend class Jdb_tcb;
  38   friend class Jdb_thread;
  39   friend class Jdb_thread_list;
  40   friend class Jdb_list_threads;
  41   friend class Jdb_list_timeouts;
  42   friend class Jdb_tbuf_show;
  43
  44 public:
  45   enum Context_mode_kernel { Kernel = 0 };
  46   enum Operation
  47   {
  48     Opcode_mask = 0xffff,
  49     Op_control = 0,
  50     Op_ex_regs = 1,
  51     Op_switch  = 2,
  52     Op_stats   = 3,
  53     Op_vcpu_resume = 4,
  54     Op_register_del_irq = 5,
  55     Op_modify_senders = 6,
  56     Op_vcpu_control= 7,
  57     Op_gdt_x86 = 0x10,
  58   };
  59
  60   enum Control_flags
  61   {
  62     Ctl_set_pager       = 0x0010000,
  63     Ctl_set_scheduler   = 0x0020000,
  64     Ctl_set_mcp         = 0x0040000,
  65     Ctl_set_prio        = 0x0080000,
  66     Ctl_set_quantum     = 0x0100000,
  67     Ctl_bind_task       = 0x0200000,
  68     Ctl_alien_thread    = 0x0400000,
  69     Ctl_ux_native       = 0x0800000,
  70     Ctl_set_exc_handler = 0x1000000,
  71   };
  72
  73   enum Ex_regs_flags
  74   {
  75     Exr_cancel            = 0x10000,
  76     Exr_trigger_exception = 0x20000,
  77   };
  78
  79   enum Vcpu_ctl_flags
  80   {
  81     Vcpu_ctl_extendet_vcpu = 0x10000,
  82   };
  83
  84
  85   class Dbg_stack
  86   {
  87   public:
  88     enum { Stack_size = Config::PAGE_SIZE };
  89     void *stack_top;
  90     Dbg_stack();
  91   };
  92
  93   static Per_cpu<Dbg_stack> dbg_stack;
  94
  95 public:
  96   typedef void (Utcb_copy_func)(Thread *sender, Thread *receiver);
  97
  98   /**
  99    * Constructor.
 100    *
 101    * @param task the task the thread should reside in.
 102    * @param id user-visible thread ID of the sender.
 103    * @param init_prio initial priority.
 104    * @param mcp maximum controlled priority.
 105    *
 106    * @post state() != Thread_invalid.
 107    */
 108   Thread();
 109
 110   int handle_page_fault (Address pfa, Mword error, Mword pc,
 111       Return_frame *regs);
 112
 113 private:
 114   struct Migration_helper_info
 115   {
 116     Migration_info inf;
 117     Thread *victim;
 118   };
 119
 120   Thread(const Thread&);        ///< Default copy constructor is undefined
 121   void *operator new(size_t);   ///< Default new operator undefined
 122
 123   bool handle_sigma0_page_fault (Address pfa);
 124
 125   /**
 126    * Return to user.
 127    *
 128    * This function is the default routine run if a newly
 129    * initialized context is being switch_exec()'ed.
 130    */
 131   static void user_invoke();
 132
 133 public:
 134   static bool pagein_tcb_request(Return_frame *regs);
 135
 136   inline Mword user_ip() const;
 137   inline void user_ip(Mword);
 138
 139   inline Mword user_sp() const;
 140   inline void user_sp(Mword);
 141
 142   inline Mword user_flags() const;
 143
 144   /** nesting level in debugger (always critical) if >1 */
 145   static Per_cpu<unsigned long> nested_trap_recover;
 146   static void handle_remote_requests_irq() asm ("handle_remote_cpu_requests");
 147   static void handle_global_remote_requests_irq() asm ("ipi_remote_call");
 148
 149 protected:
 150   explicit Thread(Context_mode_kernel);
 151
 152   // Another critical TCB cache line:
 153   Thread_lock  _thread_lock;
 154
 155   // More ipc state
 156   Thread_ptr _pager;
 157   Thread_ptr _exc_handler;
 158
 159 protected:
 160   Ram_quota *_quota;
 161   Irq_base *_del_observer;
 162
 163   // debugging stuff
 164   unsigned _magic;
 165   static const unsigned magic = 0xf001c001;
 166 };
 167
 168
 169 IMPLEMENTATION:
 170
 171 #include <cassert>
 172 #include <cstdlib>              // panic()
 173 #include <cstring>
 174 #include "atomic.h"
 175 #include "entry_frame.h"
 176 #include "fpu_alloc.h"
 177 #include "globals.h"
 178 #include "kdb_ke.h"
 179 #include "kmem_alloc.h"
 180 #include "logdefs.h"
 181 #include "map_util.h"
 182 #include "ram_quota.h"
 183 #include "sched_context.h"
 184 #include "space.h"
 185 #include "std_macros.h"
 186 #include "task.h"
 187 #include "thread_state.h"
 188 #include "timeout.h"
 189
 190 FIASCO_DEFINE_KOBJ(Thread);
 191
 192 Per_cpu<unsigned long> DEFINE_PER_CPU Thread::nested_trap_recover;
 193
 194
 195 IMPLEMENT
 196 Thread::Dbg_stack::Dbg_stack()
 197 {
 198   stack_top = Kmem_alloc::allocator()->unaligned_alloc(Stack_size);
 199   if (stack_top)
 200     stack_top = (char *)stack_top + Stack_size;
 201   //printf("JDB STACK start= %p - %p\n", (char *)stack_top - Stack_size, (char *)stack_top);
 202 }
 203
 204
 205 PUBLIC inline NEEDS[Thread::thread_lock]
 206 void
 207 Thread::kill_lock()
 208 { thread_lock()->lock(); }
 209
 210
 211 PUBLIC inline
 212 void *
 213 Thread::operator new(size_t, Ram_quota *q) throw ()
 214 {
 215   void *t = Mapped_allocator::allocator()->q_unaligned_alloc(q, Config::thread_block_size);
 216   if (t)
 217     {
 218       memset(t, 0, sizeof(Thread));
 219       reinterpret_cast<Thread*>(t)->_quota = q;
 220     }
 221   return t;
 222 }
 223
 224 /** Class-specific allocator.
 225     This allocator ensures that threads are allocated at a fixed virtual
 226     address computed from their thread ID.
 227     @param id thread ID
 228     @return address of new thread control block
 229  */
 230 PRIVATE inline
 231 void *
 232 Thread::operator new(size_t, Thread *t) throw ()
 233 {
 234   // Allocate TCB in TCB space.  Actually, do not allocate anything,
 235   // just return the address.  Allocation happens on the fly in
 236   // Thread::handle_page_fault().
 237   return t;
 238 }
 239
 240
 241 PUBLIC
 242 bool
 243 Thread::bind(Task *t, User<Utcb>::Ptr utcb)
 244 {
 245   // _utcb == 0 for all kernel threads
 246   Space::Ku_mem const *u = t->find_ku_mem(utcb, sizeof(Utcb));
 247
 248   // kernel thread?
 249   if (EXPECT_FALSE(utcb && !u))
 250     return false;
 251
 252   Lock_guard<typeof(*_space.lock())> guard(_space.lock());
 253   if (_space.space())
 254     return false;
 255
 256   _space.space(t);
 257   t->inc_ref();
 258
 259   if (u)
 260     _utcb.set(utcb, u->kern_addr(utcb));
 261
 262   return true;
 263 }
 264
 265
 266 PUBLIC inline NEEDS["kdb_ke.h", "cpu_lock.h", "space.h"]
 267 bool
 268 Thread::unbind()
 269 {
 270   Task *old;
 271
 272     {
 273       Lock_guard<typeof(*_space.lock())> guard(_space.lock());
 274
 275       if (!_space.space())
 276         return true;
 277
 278       old = static_cast<Task*>(_space.space());
 279       _space.space(0);
 280
 281       Mem_space *oms = old->mem_space();
 282
 283       if (old->dec_ref())
 284         old = 0;
 285
 286       // switch to a safe page table
 287       if (Mem_space::current_mem_space(current_cpu()) == oms)
 288         Mem_space::kernel_space()->switchin_context(oms);
 289     }
 290
 291   if (old)
 292     {
 293       current()->rcu_wait();
 294       delete old;
 295     }
 296
 297   return true;
 298 }
 299
 300 /** Cut-down version of Thread constructor; only for kernel threads
 301     Do only what's necessary to get a kernel thread started --
 302     skip all fancy stuff, no locking is necessary.
 303     @param task the address space
 304     @param id user-visible thread ID of the sender
 305  */
 306 IMPLEMENT inline
 307 Thread::Thread(Context_mode_kernel)
 308   : Receiver(), Sender(), _del_observer(0), _magic(magic)
 309 {
 310   *reinterpret_cast<void(**)()>(--_kernel_sp) = user_invoke;
 311
 312   inc_ref();
 313
 314   if (Config::stack_depth)
 315     std::memset((char*)this + sizeof(Thread), '5',
 316                 Config::thread_block_size-sizeof(Thread)-64);
 317 }
 318
 319
 320 /** Destructor.  Reestablish the Context constructor's precondition.
 321     @pre current() == thread_lock()->lock_owner()
 322          && state() == Thread_dead
 323     @pre lock_cnt() == 0
 324     @post (_kernel_sp == 0)  &&  (* (stack end) == 0)  &&  !exists()
 325  */
 326 PUBLIC virtual
 327 Thread::~Thread()               // To be called in locked state.
 328 {
 329
 330   unsigned long *init_sp = reinterpret_cast<unsigned long*>
 331     (reinterpret_cast<unsigned long>(this) + size - sizeof(Entry_frame));
 332
 333
 334   _kernel_sp = 0;
 335   *--init_sp = 0;
 336   Fpu_alloc::free_state(fpu_state());
 337   _state = Thread_invalid;
 338 }
 339
 340
 341 // IPC-gate deletion stuff ------------------------------------
 342
 343 PUBLIC inline
 344 void
 345 Thread::ipc_gate_deleted(Mword id)
 346 {
 347   (void) id;
 348   Lock_guard<Cpu_lock> g(&cpu_lock);
 349   if (_del_observer)
 350     _del_observer->hit();
 351 }
 352
 353 class Del_irq_pin : public Irq_pin_dummy
 354 {
 355 };
 356
 357 PUBLIC inline
 358 Del_irq_pin::Del_irq_pin(Thread *o)
 359 { payload()[0] = (Address)o; }
 360
 361 PUBLIC inline
 362 Thread *
 363 Del_irq_pin::thread() const
 364 { return (Thread*)payload()[0]; }
 365
 366 PUBLIC inline
 367 void
 368 Del_irq_pin::unbind_irq()
 369 { thread()->remove_delete_irq(); }
 370
 371 PUBLIC inline
 372 Del_irq_pin::~Del_irq_pin()
 373 { unbind_irq(); }
 374
 375 PUBLIC
 376 void
 377 Thread::register_delete_irq(Irq_base *irq)
 378 {
 379   irq->pin()->unbind_irq();
 380   irq->pin()->replace<Del_irq_pin>(this);
 381   _del_observer = irq;
 382 }
 383
 384 PUBLIC
 385 void
 386 Thread::remove_delete_irq()
 387 {
 388   if (!_del_observer)
 389     return;
 390
 391   Irq_base *tmp = _del_observer;
 392   _del_observer = 0;
 393   tmp->pin()->unbind_irq();
 394 }
 395
 396 // end of: IPC-gate deletion stuff -------------------------------
 397
 398
 399 /** Currently executing thread.
 400     @return currently executing thread.
 401  */
 402 inline
 403 Thread*
 404 current_thread()
 405 { return nonull_static_cast<Thread*>(current()); }
 406
 407 PUBLIC inline
 408 bool
 409 Thread::exception_triggered() const
 410 { return _exc_cont.valid(); }
 411
 412 //
 413 // state requests/manipulation
 414 //
 415
 416
 417 /** Thread lock.
 418     Overwrite Context's version of thread_lock() with a semantically
 419     equivalent, but more efficient version.
 420     @return lock used to synchronize accesses to the thread.
 421  */
 422 PUBLIC inline
 423 Thread_lock *
 424 Thread::thread_lock()
 425 { return &_thread_lock; }
 426
 427
 428 PUBLIC inline NEEDS ["config.h", "timeout.h"]
 429 void
 430 Thread::handle_timer_interrupt()
 431 {
 432   unsigned _cpu = cpu(true);
 433   // XXX: This assumes periodic timers (i.e. bogus in one-shot mode)
 434   if (!Config::fine_grained_cputime)
 435     consume_time(Config::scheduler_granularity);
 436
 437   bool resched = Rcu::do_pending_work(_cpu);
 438
 439   // Check if we need to reschedule due to timeouts or wakeups
 440   if ((Timeout_q::timeout_queue.cpu(_cpu).do_timeouts() || resched)
 441       && !schedule_in_progress())
 442     {
 443       schedule();
 444       assert (timeslice_timeout.cpu(cpu(true))->is_set());      // Coma check
 445     }
 446 }
 447
 448
 449 PUBLIC
 450 void
 451 Thread::halt()
 452 {
 453   // Cancel must be cleared on all kernel entry paths. See slowtraps for
 454   // why we delay doing it until here.
 455   state_del(Thread_cancel);
 456
 457   // we haven't been re-initialized (cancel was not set) -- so sleep
 458   if (state_change_safely(~Thread_ready, Thread_cancel | Thread_dead))
 459     while (! (state() & Thread_ready))
 460       schedule();
 461 }
 462
 463 PUBLIC static
 464 void
 465 Thread::halt_current ()
 466 {
 467   for (;;)
 468     {
 469       current_thread()->halt();
 470       kdb_ke("Thread not halted");
 471     }
 472 }
 473
 474 PRIVATE static inline
 475 void
 476 Thread::user_invoke_generic()
 477 {
 478   Context *const c = current();
 479   assert_kdb (c->state() & Thread_ready_mask);
 480
 481   if (c->handle_drq() && !c->schedule_in_progress())
 482     c->schedule();
 483
 484   // release CPU lock explicitly, because
 485   // * the context that switched to us holds the CPU lock
 486   // * we run on a newly-created stack without a CPU lock guard
 487   cpu_lock.clear();
 488 }
 489
 490
 491 PRIVATE static void
 492 Thread::leave_and_kill_myself()
 493 {
 494   current_thread()->do_kill();
 495 #ifdef CONFIG_JDB
 496   WARN("dead thread scheduled: %lx\n", current_thread()->dbg_id());
 497 #endif
 498   kdb_ke("DEAD SCHED");
 499 }
 500
 501 PUBLIC static
 502 unsigned
 503 Thread::handle_kill_helper(Drq *src, Context *, void *)
 504 {
 505   delete nonull_static_cast<Thread*>(src->context());
 506   return Drq::No_answer | Drq::Need_resched;
 507 }
 508
 509
 510 PRIVATE
 511 bool
 512 Thread::do_kill()
 513 {
 514   Lock_guard<Thread_lock> guard(thread_lock());
 515
 516   if (state() == Thread_invalid)
 517     return false;
 518
 519   //
 520   // Kill this thread
 521   //
 522
 523   // But first prevent it from being woken up by asynchronous events
 524
 525   {
 526     Lock_guard <Cpu_lock> guard(&cpu_lock);
 527
 528     // if IPC timeout active, reset it
 529     if (_timeout)
 530       _timeout->reset();
 531
 532     // Switch to time-sharing mode
 533     set_mode(Sched_mode(0));
 534
 535     // Switch to time-sharing scheduling context
 536     if (sched() != sched_context())
 537       switch_sched(sched_context());
 538
 539     if (!current_sched() || current_sched()->context() == this)
 540       set_current_sched(current()->sched());
 541   }
 542
 543   // possibly dequeue from a wait queue
 544   wait_queue_kill();
 545
 546   // if other threads want to send me IPC messages, abort these
 547   // operations
 548   {
 549     Lock_guard <Cpu_lock> guard(&cpu_lock);
 550     while (Sender *s = Sender::cast(sender_list()->head()))
 551       {
 552         s->ipc_receiver_aborted();
 553         Proc::preemption_point();
 554       }
 555   }
 556
 557   // if engaged in IPC operation, stop it
 558   if (receiver())
 559     sender_dequeue(receiver()->sender_list());
 560
 561   Context::do_kill();
 562
 563   vcpu_update_state();
 564
 565   unbind();
 566   vcpu_set_user_space(0);
 567
 568   cpu_lock.lock();
 569
 570   state_change_dirty(0, Thread_dead);
 571
 572   // dequeue from system queues
 573   ready_dequeue();
 574
 575   if (_del_observer)
 576     {
 577       _del_observer->pin()->unbind_irq();
 578       _del_observer = 0;
 579     }
 580
 581   if (dec_ref())
 582     while (1)
 583       {
 584         state_del_dirty(Thread_ready_mask);
 585         schedule();
 586         WARN("woken up dead thread %lx\n", dbg_id());
 587         kdb_ke("X");
 588       }
 589
 590   rcu_wait();
 591
 592   state_del_dirty(Thread_ready_mask);
 593
 594   ready_dequeue();
 595
 596   kernel_context_drq(handle_kill_helper, 0);
 597   kdb_ke("Im dead");
 598   return true;
 599 }
 600
 601 PRIVATE static
 602 unsigned
 603 Thread::handle_remote_kill(Drq *, Context *self, void *)
 604 {
 605   Thread *c = nonull_static_cast<Thread*>(self);
 606   c->state_add_dirty(Thread_cancel | Thread_ready);
 607   c->_exc_cont.restore(c->regs());
 608   c->do_trigger_exception(c->regs(), (void*)&Thread::leave_and_kill_myself);
 609   return 0;
 610 }
 611
 612
 613 PROTECTED
 614 bool
 615 Thread::kill()
 616 {
 617   Lock_guard<Cpu_lock> guard(&cpu_lock);
 618   inc_ref();
 619
 620
 621   if (cpu() == current_cpu())
 622     {
 623       state_add_dirty(Thread_cancel | Thread_ready);
 624       sched()->deblock(cpu());
 625       _exc_cont.restore(regs()); // overwrite an already triggered exception
 626       do_trigger_exception(regs(), (void*)&Thread::leave_and_kill_myself);
 627 //          current()->switch_exec (this, Helping);
 628       return true;
 629     }
 630
 631   drq(Thread::handle_remote_kill, 0, 0, Drq::Any_ctxt);
 632
 633   return true;
 634 }
 635
 636
 637 PUBLIC
 638 void
 639 Thread::set_sched_params(unsigned prio, Unsigned64 quantum)
 640 {
 641   Sched_context *sc = sched_context();
 642   bool const change = prio != sc->prio()
 643                    || quantum != sc->quantum();
 644   bool const ready_queued = in_ready_list();
 645
 646   if (!change && (ready_queued || this == current()))
 647     return;
 648
 649   ready_dequeue();
 650
 651   sc->set_prio(prio);
 652   sc->set_quantum(quantum);
 653   sc->replenish();
 654
 655   if (sc == current_sched())
 656     set_current_sched(sc);
 657
 658   if (state() & Thread_ready_mask)
 659     {
 660       if (this != current())
 661         ready_enqueue();
 662       else
 663         schedule();
 664     }
 665 }
 666
 667 PUBLIC
 668 long
 669 Thread::control(Thread_ptr const &pager, Thread_ptr const &exc_handler)
 670 {
 671   if (pager.is_valid())
 672     _pager = pager;
 673
 674   if (exc_handler.is_valid())
 675     _exc_handler = exc_handler;
 676
 677   return 0;
 678 }
 679
 680
 681 PRIVATE static inline
 682 bool FIASCO_WARN_RESULT
 683 Thread::copy_utcb_to_utcb(L4_msg_tag const &tag, Thread *snd, Thread *rcv,
 684                           unsigned char rights)
 685 {
 686   assert (cpu_lock.test());
 687
 688   Utcb *snd_utcb = snd->utcb().access();
 689   Utcb *rcv_utcb = rcv->utcb().access();
 690   Mword s = tag.words();
 691   Mword r = Utcb::Max_words;
 692
 693   Mem::memcpy_mwords (rcv_utcb->values, snd_utcb->values, r < s ? r : s);
 694
 695   bool success = true;
 696   if (tag.items())
 697     success = transfer_msg_items(tag, snd, snd_utcb, rcv, rcv_utcb, rights);
 698
 699   if (tag.transfer_fpu() && rcv_utcb->inherit_fpu() && (rights & L4_fpage::W))
 700     snd->transfer_fpu(rcv);
 701
 702   return success;
 703 }
 704
 705
 706 PUBLIC inline NEEDS[Thread::copy_utcb_to_ts, Thread::copy_utcb_to_utcb,
 707                     Thread::copy_ts_to_utcb]
 708 bool FIASCO_WARN_RESULT
 709 Thread::copy_utcb_to(L4_msg_tag const &tag, Thread* receiver,
 710                      unsigned char rights)
 711 {
 712   // we cannot copy trap state to trap state!
 713   assert_kdb (!this->_utcb_handler || !receiver->_utcb_handler);
 714   if (EXPECT_FALSE(this->_utcb_handler != 0))
 715     return copy_ts_to_utcb(tag, this, receiver, rights);
 716   else if (EXPECT_FALSE(receiver->_utcb_handler != 0))
 717     return copy_utcb_to_ts(tag, this, receiver, rights);
 718   else
 719     return copy_utcb_to_utcb(tag, this, receiver, rights);
 720 }
 721
 722
 723 PUBLIC static inline
 724 bool
 725 Thread::is_tcb_address(Address a)
 726 {
 727   a &= ~(Config::thread_block_size - 1);
 728   return reinterpret_cast<Thread *>(a)->_magic == magic;
 729 }
 730
 731 PUBLIC static inline
 732 void
 733 Thread::assert_irq_entry()
 734 {
 735   assert_kdb(current_thread()->schedule_in_progress()
 736              || current_thread()->state() & (Thread_ready_mask | Thread_drq_wait | Thread_waiting));
 737 }
 738
 739
 740
 741 // ---------------------------------------------------------------------------
 742
 743 PUBLIC inline
 744 bool
 745 Thread::check_sys_ipc(unsigned flags, Thread **partner, Thread **sender,
 746                       bool *have_recv) const
 747 {
 748   if (flags & L4_obj_ref::Ipc_recv)
 749     {
 750       *sender = flags & L4_obj_ref::Ipc_open_wait ? 0 : const_cast<Thread*>(this);
 751       *have_recv = true;
 752     }
 753
 754   if (flags & L4_obj_ref::Ipc_send)
 755     *partner = const_cast<Thread*>(this);
 756
 757   // FIXME: shall be removed flags == 0 is no-op
 758   if (!flags)
 759     {
 760       *sender = const_cast<Thread*>(this);
 761       *partner = const_cast<Thread*>(this);
 762       *have_recv = true;
 763     }
 764
 765   return *have_recv || ((flags & L4_obj_ref::Ipc_send) && *partner);
 766 }
 767
 768 PUBLIC static
 769 unsigned
 770 Thread::handle_migration_helper(Drq *, Context *, void *p)
 771 {
 772   Migration_helper_info const *inf = (Migration_helper_info const *)p;
 773   return inf->victim->migration_helper(&inf->inf);
 774 }
 775
 776
 777 PRIVATE
 778 void
 779 Thread::do_migration()
 780 {
 781   assert_kdb(cpu_lock.test());
 782   assert_kdb(current_cpu() == cpu(true));
 783
 784   Migration_helper_info inf;
 785
 786     {
 787       Lock_guard<typeof(_migration_rq.affinity_lock)>
 788         g(&_migration_rq.affinity_lock);
 789       inf.inf = _migration_rq.inf;
 790       _migration_rq.pending = false;
 791       _migration_rq.in_progress = true;
 792     }
 793
 794   unsigned on_cpu = cpu();
 795
 796   if (inf.inf.cpu == ~0U)
 797     {
 798       state_add_dirty(Thread_suspended);
 799       set_sched_params(0, 0);
 800       _migration_rq.in_progress = false;
 801       return;
 802     }
 803
 804   state_del_dirty(Thread_suspended);
 805
 806   if (inf.inf.cpu == on_cpu)
 807     {
 808       // stay here
 809       set_sched_params(inf.inf.prio, inf.inf.quantum);
 810       _migration_rq.in_progress = false;
 811       return;
 812     }
 813
 814   // spill FPU state into memory before migration
 815   if (state() & Thread_fpu_owner)
 816     {
 817       if (current() != this)
 818         Fpu::enable();
 819
 820       spill_fpu();
 821       Fpu::set_owner(on_cpu, 0);
 822       Fpu::disable();
 823     }
 824
 825
 826   // if we are in the middle of the scheduler, leave it now
 827   if (schedule_in_progress() == this)
 828     reset_schedule_in_progress();
 829
 830   inf.victim = this;
 831
 832   if (current() == this && Config::Max_num_cpus > 1)
 833     kernel_context_drq(handle_migration_helper, &inf);
 834   else
 835     migration_helper(&inf.inf);
 836 }
 837
 838 PUBLIC
 839 void
 840 Thread::initiate_migration()
 841 { do_migration(); }
 842
 843 PUBLIC
 844 void
 845 Thread::finish_migration()
 846 { enqueue_timeout_again(); }
 847
 848
 849 PUBLIC
 850 void
 851 Thread::migrate(Migration_info const &info)
 852 {
 853   assert_kdb (cpu_lock.test());
 854
 855   LOG_TRACE("Thread migration", "mig", this, __thread_migration_log_fmt,
 856       Migration_log *l = tbe->payload<Migration_log>();
 857       l->state = state();
 858       l->src_cpu = cpu();
 859       l->target_cpu = info.cpu;
 860       l->user_ip = regs()->ip();
 861   );
 862
 863     {
 864       Lock_guard<typeof(_migration_rq.affinity_lock)>
 865         g(&_migration_rq.affinity_lock);
 866       _migration_rq.inf = info;
 867       _migration_rq.pending = true;
 868     }
 869
 870   unsigned cpu = this->cpu();
 871
 872   if (current_cpu() == cpu)
 873     {
 874       do_migration();
 875       return;
 876     }
 877
 878   migrate_xcpu(cpu);
 879 }
 880
 881
 882 //---------------------------------------------------------------------------
 883 IMPLEMENTATION [fpu && !ux]:
 884
 885 #include "fpu.h"
 886 #include "fpu_alloc.h"
 887 #include "fpu_state.h"
 888
 889 PUBLIC inline NEEDS ["fpu.h"]
 890 void
 891 Thread::spill_fpu()
 892 {
 893   // If we own the FPU, we should never be getting an "FPU unavailable" trap
 894   assert_kdb (Fpu::owner(cpu()) == this);
 895   assert_kdb (state() & Thread_fpu_owner);
 896   assert_kdb (fpu_state());
 897
 898   // Save the FPU state of the previous FPU owner (lazy) if applicable
 899   Fpu::save_state (fpu_state());
 900   state_del_dirty (Thread_fpu_owner);
 901 }
 902
 903
 904 /*
 905  * Handle FPU trap for this context. Assumes disabled interrupts
 906  */
 907 PUBLIC inline NEEDS [Thread::spill_fpu, "fpu_alloc.h","fpu_state.h"]
 908 int
 909 Thread::switchin_fpu(bool alloc_new_fpu = true)
 910 {
 911   unsigned cpu = this->cpu(true);
 912
 913   if (state() & Thread_vcpu_fpu_disabled)
 914     return 0;
 915
 916   // If we own the FPU, we should never be getting an "FPU unavailable" trap
 917   assert_kdb (Fpu::owner(cpu) != this);
 918
 919   // Allocate FPU state slab if we didn't already have one
 920   if (!fpu_state()->state_buffer()
 921       && (EXPECT_FALSE((!alloc_new_fpu
 922                         || (state() & Thread_alien))
 923                        || !Fpu_alloc::alloc_state(_quota, fpu_state()))))
 924     return 0;
 925
 926   // Enable the FPU before accessing it, otherwise recursive trap
 927   Fpu::enable();
 928
 929   // Save the FPU state of the previous FPU owner (lazy) if applicable
 930   if (Fpu::owner(cpu))
 931     nonull_static_cast<Thread*>(Fpu::owner(cpu))->spill_fpu();
 932
 933   // Become FPU owner and restore own FPU state
 934   Fpu::restore_state(fpu_state());
 935
 936   state_add_dirty(Thread_fpu_owner);
 937   Fpu::set_owner(cpu, this);
 938   return 1;
 939 }
 940
 941 PUBLIC inline NEEDS["fpu.h", "fpu_alloc.h"]
 942 void
 943 Thread::transfer_fpu(Thread *to)
 944 {
 945   unsigned cpu = this->cpu();
 946   if (cpu != to->cpu())
 947     return;
 948
 949   if (to->fpu_state()->state_buffer())
 950     Fpu_alloc::free_state(to->fpu_state());
 951
 952   to->fpu_state()->state_buffer(fpu_state()->state_buffer());
 953   fpu_state()->state_buffer(0);
 954
 955   assert (current() == this || current() == to);
 956
 957   Fpu::disable(); // it will be reanabled in switch_fpu
 958
 959   if (EXPECT_FALSE(Fpu::owner(cpu) == to))
 960     {
 961       assert_kdb (to->state() & Thread_fpu_owner);
 962
 963       Fpu::set_owner(cpu, 0);
 964       to->state_del_dirty (Thread_fpu_owner);
 965     }
 966   else if (Fpu::owner(cpu) == this)
 967     {
 968       assert_kdb (state() & Thread_fpu_owner);
 969
 970       state_del_dirty (Thread_fpu_owner);
 971
 972       to->state_add_dirty (Thread_fpu_owner);
 973       Fpu::set_owner(cpu, to);
 974       if (EXPECT_FALSE(current() == to))
 975         Fpu::enable();
 976     }
 977 }
 978
 979 //---------------------------------------------------------------------------
 980 IMPLEMENTATION [!fpu]:
 981
 982 PUBLIC inline
 983 int
 984 Thread::switchin_fpu(bool alloc_new_fpu = true)
 985 {
 986   (void)alloc_new_fpu;
 987   return 0;
 988 }
 989
 990 PUBLIC inline
 991 void
 992 Thread::spill_fpu()
 993 {}
 994
 995 //---------------------------------------------------------------------------
 996 IMPLEMENTATION [!fpu || ux]:
 997
 998 PUBLIC inline
 999 void
1000 Thread::transfer_fpu(Thread *)
1001 {}
1002
1003 //---------------------------------------------------------------------------
1004 IMPLEMENTATION [!log]:
1005
1006 PUBLIC inline
1007 unsigned Thread::sys_ipc_log(Syscall_frame *)
1008 { return 0; }
1009
1010 PUBLIC inline
1011 unsigned Thread::sys_ipc_trace(Syscall_frame *)
1012 { return 0; }
1013
1014 static inline
1015 void Thread::page_fault_log(Address, unsigned, unsigned)
1016 {}
1017
1018 PUBLIC static inline
1019 int Thread::log_page_fault()
1020 { return 0; }
1021
1022 PUBLIC inline
1023 unsigned Thread::sys_fpage_unmap_log(Syscall_frame *)
1024 { return 0; }
1025
1026
1027 // ----------------------------------------------------------------------------
1028 IMPLEMENTATION [!mp]:
1029
1030
1031 PRIVATE inline
1032 unsigned
1033 Thread::migration_helper(Migration_info const *inf)
1034 {
1035   unsigned cpu = inf->cpu;
1036   //  LOG_MSG_3VAL(this, "MGi ", Mword(current()), (current_cpu() << 16) | cpu(), Context::current_sched());
1037   if (_timeout)
1038     _timeout->reset();
1039   ready_dequeue();
1040
1041     {
1042       // Not sure if this can ever happen
1043       Sched_context *csc = Context::current_sched();
1044       if (!csc || csc->context() == this)
1045         Context::set_current_sched(current()->sched());
1046     }
1047
1048   Sched_context *sc = sched_context();
1049   sc->set_prio(inf->prio);
1050   sc->set_quantum(inf->quantum);
1051   sc->replenish();
1052   set_sched(sc);
1053
1054   if (drq_pending())
1055     state_add_dirty(Thread_drq_ready);
1056
1057   set_cpu_of(this, cpu);
1058   return  Drq::No_answer | Drq::Need_resched;
1059 }
1060
1061 PRIVATE inline
1062 void
1063 Thread::migrate_xcpu(unsigned cpu)
1064 {
1065   (void)cpu;
1066   assert_kdb (false);
1067 }
1068
1069
1070 //----------------------------------------------------------------------------
1071 INTERFACE [debug]:
1072
1073 EXTENSION class Thread
1074 {
1075 protected:
1076   struct Migration_log
1077   {
1078     Mword    state;
1079     Address  user_ip;
1080     unsigned src_cpu;
1081     unsigned target_cpu;
1082
1083     static unsigned fmt(Tb_entry *, int, char *)
1084     asm ("__thread_migration_log_fmt");
1085   };
1086 };
1087
1088
1089 // ----------------------------------------------------------------------------
1090 IMPLEMENTATION [mp]:
1091
1092 #include "ipi.h"
1093
1094 IMPLEMENT
1095 void
1096 Thread::handle_remote_requests_irq()
1097 {
1098   assert_kdb (cpu_lock.test());
1099   // printf("CPU[%2u]: > RQ IPI (current=%p)\n", current_cpu(), current());
1100   Ipi::eoi(Ipi::Request);
1101   Context *const c = current();
1102   //LOG_MSG_3VAL(c, "ipi", c->cpu(), (Mword)c, c->drq_pending());
1103   Context *migration_q = 0;
1104   bool resched = _pending_rqq.cpu(c->cpu()).handle_requests(&migration_q);
1105
1106   resched |= Rcu::do_pending_work(c->cpu());
1107
1108   if (migration_q)
1109     static_cast<Thread*>(migration_q)->do_migration();
1110
1111   if ((resched || c->handle_drq()) && !c->schedule_in_progress())
1112     {
1113       //LOG_MSG_3VAL(c, "ipis", 0, 0, 0);
1114       // printf("CPU[%2u]: RQ IPI sched %p\n", current_cpu(), current());
1115       c->schedule();
1116     }
1117   // printf("CPU[%2u]: < RQ IPI (current=%p)\n", current_cpu(), current());
1118 }
1119
1120 IMPLEMENT
1121 void
1122 Thread::handle_global_remote_requests_irq()
1123 {
1124   assert_kdb (cpu_lock.test());
1125   // printf("CPU[%2u]: > RQ IPI (current=%p)\n", current_cpu(), current());
1126   Ipi::eoi(Ipi::Global_request);
1127   Context::handle_global_requests();
1128 }
1129
1130 PRIVATE inline
1131 unsigned
1132 Thread::migration_helper(Migration_info const *inf)
1133 {
1134   // LOG_MSG_3VAL(this, "MGi ", Mword(current()), (current_cpu() << 16) | cpu(), 0);
1135   assert_kdb (cpu() == current_cpu());
1136   assert_kdb (current() != this);
1137   assert_kdb (cpu_lock.test());
1138
1139   if (_timeout)
1140     _timeout->reset();
1141   ready_dequeue();
1142
1143     {
1144       // Not sure if this can ever happen
1145       Sched_context *csc = Context::current_sched();
1146       if (!csc || csc->context() == this)
1147         Context::set_current_sched(current()->sched());
1148     }
1149
1150   unsigned cpu = inf->cpu;
1151
1152     {
1153       Queue &q = _pending_rqq.cpu(current_cpu());
1154       // The queue lock of the current CPU protects the cpu number in
1155       // the thread
1156       Lock_guard<Pending_rqq::Inner_lock> g(q.q_lock());
1157
1158       // potentailly dequeue from our local queue
1159       if (_pending_rq.queued())
1160         check_kdb (q.dequeue(&_pending_rq, Queue_item::Ok));
1161
1162       Sched_context *sc = sched_context();
1163       sc->set_prio(inf->prio);
1164       sc->set_quantum(inf->quantum);
1165       sc->replenish();
1166       set_sched(sc);
1167
1168       if (drq_pending())
1169         state_add_dirty(Thread_drq_ready);
1170
1171       Mem::mp_wmb();
1172
1173       assert_kdb (!in_ready_list());
1174
1175       set_cpu_of(this, cpu);
1176       // now we are migrated away fom current_cpu
1177     }
1178
1179   bool ipi = true;
1180
1181     {
1182       Queue &q = _pending_rqq.cpu(cpu);
1183       Lock_guard<Pending_rqq::Inner_lock> g(q.q_lock());
1184
1185       // migrated meanwhile
1186       if (this->cpu() != cpu || _pending_rq.queued())
1187         return  Drq::No_answer | Drq::Need_resched;
1188
1189       if (q.first())
1190         ipi = false;
1191
1192       q.enqueue(&_pending_rq);
1193     }
1194
1195   if (ipi)
1196     {
1197       //LOG_MSG_3VAL(this, "sipi", current_cpu(), cpu(), (Mword)current());
1198       Ipi::cpu(cpu).send(Ipi::Request);
1199     }
1200
1201   return  Drq::No_answer | Drq::Need_resched;
1202 }
1203
1204 PRIVATE inline
1205 void
1206 Thread::migrate_xcpu(unsigned cpu)
1207 {
1208   bool ipi = true;
1209
1210     {
1211       Queue &q = Context::_pending_rqq.cpu(cpu);
1212       Lock_guard<Pending_rqq::Inner_lock> g(q.q_lock());
1213
1214       // already migrated
1215       if (cpu != this->cpu())
1216         return;
1217
1218       if (q.first())
1219         ipi = false;
1220
1221       if (!_pending_rq.queued())
1222         q.enqueue(&_pending_rq);
1223       else
1224         ipi = false;
1225     }
1226
1227   if (ipi)
1228     Ipi::cpu(cpu).send(Ipi::Request);
1229 }
1230
1231 //----------------------------------------------------------------------------
1232 IMPLEMENTATION [debug]:
1233
1234 IMPLEMENT
1235 unsigned
1236 Thread::Migration_log::fmt(Tb_entry *e, int maxlen, char *buf)
1237 {
1238   Migration_log *l = e->payload<Migration_log>();
1239   return snprintf(buf, maxlen, "migrate from %u to %u (state=%lx user ip=%lx)",
1240       l->src_cpu, l->target_cpu, l->state, l->user_ip);
1241 }
1242