kernel/fiasco/src/kern/thread.cpp

   1 INTERFACE:
   2
   3 #include "l4_types.h"
   4 #include "config.h"
   5 #include "continuation.h"
   6 #include "helping_lock.h"
   7 #include "kobject.h"
   8 #include "mem_layout.h"
   9 #include "member_offs.h"
  10 #include "receiver.h"
  11 #include "ref_obj.h"
  12 #include "sender.h"
  13 #include "space.h"              // Space_index
  14 #include "spin_lock.h"
  15 #include "thread_lock.h"
  16
  17 class Return_frame;
  18 class Syscall_frame;
  19 class Vcpu_state;
  20 class Irq_base;
  21
  22 typedef Context_ptr_base<Thread> Thread_ptr;
  23
  24
  25 /** A thread.  This class is the driver class for most kernel functionality.
  26  */
  27 class Thread :
  28   public Receiver,
  29   public Sender,
  30   public Kobject
  31 {
  32   MEMBER_OFFSET();
  33   FIASCO_DECLARE_KOBJ();
  34
  35   friend class Jdb;
  36   friend class Jdb_bt;
  37   friend class Jdb_tcb;
  38   friend class Jdb_thread;
  39   friend class Jdb_thread_list;
  40   friend class Jdb_list_threads;
  41   friend class Jdb_list_timeouts;
  42   friend class Jdb_tbuf_show;
  43
  44 public:
  45   enum Context_mode_kernel { Kernel = 0 };
  46   enum Operation
  47   {
  48     Opcode_mask = 0xffff,
  49     Op_control = 0,
  50     Op_ex_regs = 1,
  51     Op_switch  = 2,
  52     Op_stats   = 3,
  53     Op_vcpu_resume = 4,
  54     Op_register_del_irq = 5,
  55     Op_modify_senders = 6,
  56     Op_vcpu_control= 7,
  57     Op_gdt_x86 = 0x10,
  58   };
  59
  60   enum Control_flags
  61   {
  62     Ctl_set_pager       = 0x0010000,
  63     Ctl_set_scheduler   = 0x0020000,
  64     Ctl_set_mcp         = 0x0040000,
  65     Ctl_set_prio        = 0x0080000,
  66     Ctl_set_quantum     = 0x0100000,
  67     Ctl_bind_task       = 0x0200000,
  68     Ctl_alien_thread    = 0x0400000,
  69     Ctl_ux_native       = 0x0800000,
  70     Ctl_set_exc_handler = 0x1000000,
  71   };
  72
  73   enum Ex_regs_flags
  74   {
  75     Exr_cancel            = 0x10000,
  76     Exr_trigger_exception = 0x20000,
  77   };
  78
  79   enum Vcpu_ctl_flags
  80   {
  81     Vcpu_ctl_extendet_vcpu = 0x10000,
  82   };
  83
  84
  85   class Dbg_stack
  86   {
  87   public:
  88     enum { Stack_size = Config::PAGE_SIZE };
  89     void *stack_top;
  90     Dbg_stack();
  91   };
  92
  93   static Per_cpu<Dbg_stack> dbg_stack;
  94
  95 public:
  96   typedef void (Utcb_copy_func)(Thread *sender, Thread *receiver);
  97
  98   /**
  99    * Constructor.
 100    *
 101    * @param task the task the thread should reside in.
 102    * @param id user-visible thread ID of the sender.
 103    * @param init_prio initial priority.
 104    * @param mcp maximum controlled priority.
 105    *
 106    * @post state() != Thread_invalid.
 107    */
 108   Thread();
 109
 110   int handle_page_fault (Address pfa, Mword error, Mword pc,
 111       Return_frame *regs);
 112
 113 private:
 114   struct Migration_helper_info
 115   {
 116     Migration_info inf;
 117     Thread *victim;
 118   };
 119
 120   Thread(const Thread&);        ///< Default copy constructor is undefined
 121   void *operator new(size_t);   ///< Default new operator undefined
 122
 123   bool handle_sigma0_page_fault (Address pfa);
 124
 125   /**
 126    * Return to user.
 127    *
 128    * This function is the default routine run if a newly
 129    * initialized context is being switch_exec()'ed.
 130    */
 131   static void user_invoke();
 132
 133 public:
 134   static bool pagein_tcb_request(Return_frame *regs);
 135
 136   inline Mword user_ip() const;
 137   inline void user_ip(Mword);
 138
 139   inline Mword user_sp() const;
 140   inline void user_sp(Mword);
 141
 142   inline Mword user_flags() const;
 143
 144   /** nesting level in debugger (always critical) if >1 */
 145   static Per_cpu<unsigned long> nested_trap_recover;
 146   static void handle_remote_requests_irq() asm ("handle_remote_cpu_requests");
 147   static void handle_global_remote_requests_irq() asm ("ipi_remote_call");
 148
 149 protected:
 150   explicit Thread(Context_mode_kernel);
 151
 152   // Another critical TCB cache line:
 153   Thread_lock  _thread_lock;
 154
 155   // More ipc state
 156   Thread_ptr _pager;
 157   Thread_ptr _exc_handler;
 158
 159 protected:
 160   Ram_quota *_quota;
 161   Irq_base *_del_observer;
 162
 163   // debugging stuff
 164   unsigned _magic;
 165   static const unsigned magic = 0xf001c001;
 166 };
 167
 168
 169 IMPLEMENTATION:
 170
 171 #include <cassert>
 172 #include <cstdlib>              // panic()
 173 #include <cstring>
 174 #include "atomic.h"
 175 #include "entry_frame.h"
 176 #include "fpu_alloc.h"
 177 #include "globals.h"
 178 #include "kdb_ke.h"
 179 #include "kmem_alloc.h"
 180 #include "logdefs.h"
 181 #include "map_util.h"
 182 #include "ram_quota.h"
 183 #include "sched_context.h"
 184 #include "space.h"
 185 #include "std_macros.h"
 186 #include "task.h"
 187 #include "thread_state.h"
 188 #include "timeout.h"
 189
 190 FIASCO_DEFINE_KOBJ(Thread);
 191
 192 Per_cpu<unsigned long> DEFINE_PER_CPU Thread::nested_trap_recover;
 193
 194
 195 IMPLEMENT
 196 Thread::Dbg_stack::Dbg_stack()
 197 {
 198   stack_top = Kmem_alloc::allocator()->unaligned_alloc(Stack_size);
 199   if (stack_top)
 200     stack_top = (char *)stack_top + Stack_size;
 201   //printf("JDB STACK start= %p - %p\n", (char *)stack_top - Stack_size, (char *)stack_top);
 202 }
 203
 204
 205 PUBLIC inline NEEDS[Thread::thread_lock]
 206 void
 207 Thread::kill_lock()
 208 { thread_lock()->lock(); }
 209
 210
 211 PUBLIC inline
 212 void *
 213 Thread::operator new(size_t, Ram_quota *q) throw ()
 214 {
 215   void *t = Mapped_allocator::allocator()->q_unaligned_alloc(q, Config::thread_block_size);
 216   if (t)
 217     {
 218       memset(t, 0, sizeof(Thread));
 219       reinterpret_cast<Thread*>(t)->_quota = q;
 220     }
 221   return t;
 222 }
 223
 224 /** Class-specific allocator.
 225     This allocator ensures that threads are allocated at a fixed virtual
 226     address computed from their thread ID.
 227     @param id thread ID
 228     @return address of new thread control block
 229  */
 230 PRIVATE inline
 231 void *
 232 Thread::operator new(size_t, Thread *t) throw ()
 233 {
 234   // Allocate TCB in TCB space.  Actually, do not allocate anything,
 235   // just return the address.  Allocation happens on the fly in
 236   // Thread::handle_page_fault().
 237   return t;
 238 }
 239
 240
 241 PUBLIC
 242 bool
 243 Thread::bind(Task *t, User<Utcb>::Ptr utcb)
 244 {
 245   // _utcb == 0 for all kernel threads
 246   Space::Ku_mem const *u = t->find_ku_mem(utcb, sizeof(Utcb));
 247
 248   // kernel thread?
 249   if (EXPECT_FALSE(utcb && !u))
 250     return false;
 251
 252   Lock_guard<typeof(*_space.lock())> guard(_space.lock());
 253   if (_space.space())
 254     return false;
 255
 256   _space.space(t);
 257   t->inc_ref();
 258
 259   if (u)
 260     _utcb.set(utcb, u->kern_addr(utcb));
 261
 262   return true;
 263 }
 264
 265
 266 PUBLIC inline NEEDS["kdb_ke.h", "cpu_lock.h", "space.h"]
 267 bool
 268 Thread::unbind()
 269 {
 270   Task *old;
 271
 272     {
 273       Lock_guard<typeof(*_space.lock())> guard(_space.lock());
 274
 275       if (!_space.space())
 276         return true;
 277
 278       old = static_cast<Task*>(_space.space());
 279       _space.space(0);
 280
 281       Mem_space *oms = old->mem_space();
 282
 283       if (old->dec_ref())
 284         old = 0;
 285
 286       // switch to a safe page table
 287       if (Mem_space::current_mem_space(current_cpu()) == oms)
 288         Mem_space::kernel_space()->switchin_context(oms);
 289     }
 290
 291   if (old)
 292     {
 293       current()->rcu_wait();
 294       delete old;
 295     }
 296
 297   return true;
 298 }
 299
 300 /** Cut-down version of Thread constructor; only for kernel threads
 301     Do only what's necessary to get a kernel thread started --
 302     skip all fancy stuff, no locking is necessary.
 303     @param task the address space
 304     @param id user-visible thread ID of the sender
 305  */
 306 IMPLEMENT inline
 307 Thread::Thread(Context_mode_kernel)
 308   : Receiver(), Sender(), _del_observer(0), _magic(magic)
 309 {
 310   *reinterpret_cast<void(**)()>(--_kernel_sp) = user_invoke;
 311
 312   inc_ref();
 313
 314   if (Config::stack_depth)
 315     std::memset((char*)this + sizeof(Thread), '5',
 316                 Config::thread_block_size-sizeof(Thread)-64);
 317 }
 318
 319
 320 /** Destructor.  Reestablish the Context constructor's precondition.
 321     @pre current() == thread_lock()->lock_owner()
 322          && state() == Thread_dead
 323     @pre lock_cnt() == 0
 324     @post (_kernel_sp == 0)  &&  (* (stack end) == 0)  &&  !exists()
 325  */
 326 PUBLIC virtual
 327 Thread::~Thread()               // To be called in locked state.
 328 {
 329
 330   unsigned long *init_sp = reinterpret_cast<unsigned long*>
 331     (reinterpret_cast<unsigned long>(this) + size - sizeof(Entry_frame));
 332
 333
 334   _kernel_sp = 0;
 335   *--init_sp = 0;
 336   Fpu_alloc::free_state(fpu_state());
 337   _state = Thread_invalid;
 338 }
 339
 340
 341 // IPC-gate deletion stuff ------------------------------------
 342
 343 PUBLIC inline
 344 void
 345 Thread::ipc_gate_deleted(Mword id)
 346 {
 347   (void) id;
 348   Lock_guard<Cpu_lock> g(&cpu_lock);
 349   if (_del_observer)
 350     _del_observer->hit();
 351 }
 352
 353 class Del_irq_pin : public Irq_pin_dummy
 354 {
 355 };
 356
 357 PUBLIC inline
 358 Del_irq_pin::Del_irq_pin(Thread *o)
 359 { payload()[0] = (Address)o; }
 360
 361 PUBLIC inline
 362 Thread *
 363 Del_irq_pin::thread() const
 364 { return (Thread*)payload()[0]; }
 365
 366 PUBLIC inline
 367 void
 368 Del_irq_pin::unbind_irq()
 369 { thread()->remove_delete_irq(); }
 370
 371 PUBLIC inline
 372 Del_irq_pin::~Del_irq_pin()
 373 { unbind_irq(); }
 374
 375 PUBLIC
 376 void
 377 Thread::register_delete_irq(Irq_base *irq)
 378 {
 379   irq->pin()->unbind_irq();
 380   irq->pin()->replace<Del_irq_pin>(this);
 381   _del_observer = irq;
 382 }
 383
 384 PUBLIC
 385 void
 386 Thread::remove_delete_irq()
 387 {
 388   if (!_del_observer)
 389     return;
 390
 391   Irq_base *tmp = _del_observer;
 392   _del_observer = 0;
 393   tmp->pin()->unbind_irq();
 394 }
 395
 396 // end of: IPC-gate deletion stuff -------------------------------
 397
 398
 399 /** Currently executing thread.
 400     @return currently executing thread.
 401  */
 402 inline
 403 Thread*
 404 current_thread()
 405 { return nonull_static_cast<Thread*>(current()); }
 406
 407 PUBLIC inline
 408 bool
 409 Thread::exception_triggered() const
 410 { return _exc_cont.valid(); }
 411
 412 PUBLIC inline
 413 bool
 414 Thread::continuation_test_and_restore()
 415 {
 416   bool v = _exc_cont.valid();
 417   if (v)
 418     _exc_cont.restore(regs());
 419   return v;
 420 }
 421
 422 //
 423 // state requests/manipulation
 424 //
 425
 426
 427 /** Thread lock.
 428     Overwrite Context's version of thread_lock() with a semantically
 429     equivalent, but more efficient version.
 430     @return lock used to synchronize accesses to the thread.
 431  */
 432 PUBLIC inline
 433 Thread_lock *
 434 Thread::thread_lock()
 435 { return &_thread_lock; }
 436
 437
 438 PUBLIC inline NEEDS ["config.h", "timeout.h"]
 439 void
 440 Thread::handle_timer_interrupt()
 441 {
 442   unsigned _cpu = cpu(true);
 443   // XXX: This assumes periodic timers (i.e. bogus in one-shot mode)
 444   if (!Config::fine_grained_cputime)
 445     consume_time(Config::scheduler_granularity);
 446
 447   bool resched = Rcu::do_pending_work(_cpu);
 448
 449   // Check if we need to reschedule due to timeouts or wakeups
 450   if ((Timeout_q::timeout_queue.cpu(_cpu).do_timeouts() || resched)
 451       && !schedule_in_progress())
 452     {
 453       schedule();
 454       assert (timeslice_timeout.cpu(cpu(true))->is_set());      // Coma check
 455     }
 456 }
 457
 458
 459 PUBLIC
 460 void
 461 Thread::halt()
 462 {
 463   // Cancel must be cleared on all kernel entry paths. See slowtraps for
 464   // why we delay doing it until here.
 465   state_del(Thread_cancel);
 466
 467   // we haven't been re-initialized (cancel was not set) -- so sleep
 468   if (state_change_safely(~Thread_ready, Thread_cancel | Thread_dead))
 469     while (! (state() & Thread_ready))
 470       schedule();
 471 }
 472
 473 PUBLIC static
 474 void
 475 Thread::halt_current ()
 476 {
 477   for (;;)
 478     {
 479       current_thread()->halt();
 480       kdb_ke("Thread not halted");
 481     }
 482 }
 483
 484 PRIVATE static inline
 485 void
 486 Thread::user_invoke_generic()
 487 {
 488   Context *const c = current();
 489   assert_kdb (c->state() & Thread_ready_mask);
 490
 491   if (c->handle_drq() && !c->schedule_in_progress())
 492     c->schedule();
 493
 494   // release CPU lock explicitly, because
 495   // * the context that switched to us holds the CPU lock
 496   // * we run on a newly-created stack without a CPU lock guard
 497   cpu_lock.clear();
 498 }
 499
 500
 501 PRIVATE static void
 502 Thread::leave_and_kill_myself()
 503 {
 504   current_thread()->do_kill();
 505 #ifdef CONFIG_JDB
 506   WARN("dead thread scheduled: %lx\n", current_thread()->dbg_id());
 507 #endif
 508   kdb_ke("DEAD SCHED");
 509 }
 510
 511 PUBLIC static
 512 unsigned
 513 Thread::handle_kill_helper(Drq *src, Context *, void *)
 514 {
 515   delete nonull_static_cast<Thread*>(src->context());
 516   return Drq::No_answer | Drq::Need_resched;
 517 }
 518
 519
 520 PRIVATE
 521 bool
 522 Thread::do_kill()
 523 {
 524   Lock_guard<Thread_lock> guard(thread_lock());
 525
 526   if (state() == Thread_invalid)
 527     return false;
 528
 529   //
 530   // Kill this thread
 531   //
 532
 533   // But first prevent it from being woken up by asynchronous events
 534
 535   {
 536     Lock_guard <Cpu_lock> guard(&cpu_lock);
 537
 538     // if IPC timeout active, reset it
 539     if (_timeout)
 540       _timeout->reset();
 541
 542     // Switch to time-sharing mode
 543     set_mode(Sched_mode(0));
 544
 545     // Switch to time-sharing scheduling context
 546     if (sched() != sched_context())
 547       switch_sched(sched_context());
 548
 549     if (!current_sched() || current_sched()->context() == this)
 550       set_current_sched(current()->sched());
 551   }
 552
 553   // possibly dequeue from a wait queue
 554   wait_queue_kill();
 555
 556   // if other threads want to send me IPC messages, abort these
 557   // operations
 558   {
 559     Lock_guard <Cpu_lock> guard(&cpu_lock);
 560     while (Sender *s = Sender::cast(sender_list()->head()))
 561       {
 562         s->ipc_receiver_aborted();
 563         Proc::preemption_point();
 564       }
 565   }
 566
 567   // if engaged in IPC operation, stop it
 568   if (receiver())
 569     sender_dequeue(receiver()->sender_list());
 570
 571   Context::do_kill();
 572
 573   vcpu_update_state();
 574
 575   unbind();
 576   vcpu_set_user_space(0);
 577
 578   cpu_lock.lock();
 579
 580   state_change_dirty(0, Thread_dead);
 581
 582   // dequeue from system queues
 583   ready_dequeue();
 584
 585   if (_del_observer)
 586     {
 587       _del_observer->pin()->unbind_irq();
 588       _del_observer = 0;
 589     }
 590
 591   if (dec_ref())
 592     while (1)
 593       {
 594         state_del_dirty(Thread_ready_mask);
 595         schedule();
 596         WARN("woken up dead thread %lx\n", dbg_id());
 597         kdb_ke("X");
 598       }
 599
 600   rcu_wait();
 601
 602   state_del_dirty(Thread_ready_mask);
 603
 604   ready_dequeue();
 605
 606   kernel_context_drq(handle_kill_helper, 0);
 607   kdb_ke("Im dead");
 608   return true;
 609 }
 610
 611 PRIVATE static
 612 unsigned
 613 Thread::handle_remote_kill(Drq *, Context *self, void *)
 614 {
 615   Thread *c = nonull_static_cast<Thread*>(self);
 616   c->state_add_dirty(Thread_cancel | Thread_ready);
 617   c->_exc_cont.restore(c->regs());
 618   c->do_trigger_exception(c->regs(), (void*)&Thread::leave_and_kill_myself);
 619   return 0;
 620 }
 621
 622
 623 PROTECTED
 624 bool
 625 Thread::kill()
 626 {
 627   Lock_guard<Cpu_lock> guard(&cpu_lock);
 628   inc_ref();
 629
 630
 631   if (cpu() == current_cpu())
 632     {
 633       state_add_dirty(Thread_cancel | Thread_ready);
 634       sched()->deblock(cpu());
 635       _exc_cont.restore(regs()); // overwrite an already triggered exception
 636       do_trigger_exception(regs(), (void*)&Thread::leave_and_kill_myself);
 637 //          current()->switch_exec (this, Helping);
 638       return true;
 639     }
 640
 641   drq(Thread::handle_remote_kill, 0, 0, Drq::Any_ctxt);
 642
 643   return true;
 644 }
 645
 646
 647 PUBLIC
 648 void
 649 Thread::set_sched_params(unsigned prio, Unsigned64 quantum)
 650 {
 651   Sched_context *sc = sched_context();
 652   bool const change = prio != sc->prio()
 653                    || quantum != sc->quantum();
 654   bool const ready_queued = in_ready_list();
 655
 656   if (!change && (ready_queued || this == current()))
 657     return;
 658
 659   ready_dequeue();
 660
 661   sc->set_prio(prio);
 662   sc->set_quantum(quantum);
 663   sc->replenish();
 664
 665   if (sc == current_sched())
 666     set_current_sched(sc);
 667
 668   if (state() & Thread_ready_mask)
 669     {
 670       if (this != current())
 671         ready_enqueue();
 672       else
 673         schedule();
 674     }
 675 }
 676
 677 PUBLIC
 678 long
 679 Thread::control(Thread_ptr const &pager, Thread_ptr const &exc_handler)
 680 {
 681   if (pager.is_valid())
 682     _pager = pager;
 683
 684   if (exc_handler.is_valid())
 685     _exc_handler = exc_handler;
 686
 687   return 0;
 688 }
 689
 690
 691 PRIVATE static inline
 692 bool FIASCO_WARN_RESULT
 693 Thread::copy_utcb_to_utcb(L4_msg_tag const &tag, Thread *snd, Thread *rcv,
 694                           unsigned char rights)
 695 {
 696   assert (cpu_lock.test());
 697
 698   Utcb *snd_utcb = snd->utcb().access();
 699   Utcb *rcv_utcb = rcv->utcb().access();
 700   Mword s = tag.words();
 701   Mword r = Utcb::Max_words;
 702
 703   Mem::memcpy_mwords (rcv_utcb->values, snd_utcb->values, r < s ? r : s);
 704
 705   bool success = true;
 706   if (tag.items())
 707     success = transfer_msg_items(tag, snd, snd_utcb, rcv, rcv_utcb, rights);
 708
 709   if (tag.transfer_fpu() && rcv_utcb->inherit_fpu() && (rights & L4_fpage::W))
 710     snd->transfer_fpu(rcv);
 711
 712   return success;
 713 }
 714
 715
 716 PUBLIC inline NEEDS[Thread::copy_utcb_to_ts, Thread::copy_utcb_to_utcb,
 717                     Thread::copy_ts_to_utcb]
 718 bool FIASCO_WARN_RESULT
 719 Thread::copy_utcb_to(L4_msg_tag const &tag, Thread* receiver,
 720                      unsigned char rights)
 721 {
 722   // we cannot copy trap state to trap state!
 723   assert_kdb (!this->_utcb_handler || !receiver->_utcb_handler);
 724   if (EXPECT_FALSE(this->_utcb_handler != 0))
 725     return copy_ts_to_utcb(tag, this, receiver, rights);
 726   else if (EXPECT_FALSE(receiver->_utcb_handler != 0))
 727     return copy_utcb_to_ts(tag, this, receiver, rights);
 728   else
 729     return copy_utcb_to_utcb(tag, this, receiver, rights);
 730 }
 731
 732
 733 PUBLIC static inline
 734 bool
 735 Thread::is_tcb_address(Address a)
 736 {
 737   a &= ~(Config::thread_block_size - 1);
 738   return reinterpret_cast<Thread *>(a)->_magic == magic;
 739 }
 740
 741 PUBLIC static inline
 742 void
 743 Thread::assert_irq_entry()
 744 {
 745   assert_kdb(current_thread()->schedule_in_progress()
 746              || current_thread()->state() & (Thread_ready_mask | Thread_drq_wait | Thread_waiting));
 747 }
 748
 749
 750
 751 // ---------------------------------------------------------------------------
 752
 753 PUBLIC inline
 754 bool
 755 Thread::check_sys_ipc(unsigned flags, Thread **partner, Thread **sender,
 756                       bool *have_recv) const
 757 {
 758   if (flags & L4_obj_ref::Ipc_recv)
 759     {
 760       *sender = flags & L4_obj_ref::Ipc_open_wait ? 0 : const_cast<Thread*>(this);
 761       *have_recv = true;
 762     }
 763
 764   if (flags & L4_obj_ref::Ipc_send)
 765     *partner = const_cast<Thread*>(this);
 766
 767   // FIXME: shall be removed flags == 0 is no-op
 768   if (!flags)
 769     {
 770       *sender = const_cast<Thread*>(this);
 771       *partner = const_cast<Thread*>(this);
 772       *have_recv = true;
 773     }
 774
 775   return *have_recv || ((flags & L4_obj_ref::Ipc_send) && *partner);
 776 }
 777
 778 PUBLIC static
 779 unsigned
 780 Thread::handle_migration_helper(Drq *, Context *, void *p)
 781 {
 782   Migration_helper_info const *inf = (Migration_helper_info const *)p;
 783   return inf->victim->migration_helper(&inf->inf);
 784 }
 785
 786
 787 PRIVATE
 788 void
 789 Thread::do_migration()
 790 {
 791   assert_kdb(cpu_lock.test());
 792   assert_kdb(current_cpu() == cpu(true));
 793
 794   Migration_helper_info inf;
 795
 796     {
 797       Lock_guard<typeof(_migration_rq.affinity_lock)>
 798         g(&_migration_rq.affinity_lock);
 799       inf.inf = _migration_rq.inf;
 800       _migration_rq.pending = false;
 801       _migration_rq.in_progress = true;
 802     }
 803
 804   unsigned on_cpu = cpu();
 805
 806   if (inf.inf.cpu == ~0U)
 807     {
 808       state_add_dirty(Thread_suspended);
 809       set_sched_params(0, 0);
 810       _migration_rq.in_progress = false;
 811       return;
 812     }
 813
 814   state_del_dirty(Thread_suspended);
 815
 816   if (inf.inf.cpu == on_cpu)
 817     {
 818       // stay here
 819       set_sched_params(inf.inf.prio, inf.inf.quantum);
 820       _migration_rq.in_progress = false;
 821       return;
 822     }
 823
 824   // spill FPU state into memory before migration
 825   if (state() & Thread_fpu_owner)
 826     {
 827       if (current() != this)
 828         Fpu::enable();
 829
 830       spill_fpu();
 831       Fpu::set_owner(on_cpu, 0);
 832       Fpu::disable();
 833     }
 834
 835
 836   // if we are in the middle of the scheduler, leave it now
 837   if (schedule_in_progress() == this)
 838     reset_schedule_in_progress();
 839
 840   inf.victim = this;
 841
 842   if (current() == this && Config::Max_num_cpus > 1)
 843     kernel_context_drq(handle_migration_helper, &inf);
 844   else
 845     migration_helper(&inf.inf);
 846 }
 847
 848 PUBLIC
 849 void
 850 Thread::initiate_migration()
 851 { do_migration(); }
 852
 853 PUBLIC
 854 void
 855 Thread::finish_migration()
 856 { enqueue_timeout_again(); }
 857
 858
 859 PUBLIC
 860 void
 861 Thread::migrate(Migration_info const &info)
 862 {
 863   assert_kdb (cpu_lock.test());
 864
 865   LOG_TRACE("Thread migration", "mig", this, __thread_migration_log_fmt,
 866       Migration_log *l = tbe->payload<Migration_log>();
 867       l->state = state();
 868       l->src_cpu = cpu();
 869       l->target_cpu = info.cpu;
 870       l->user_ip = regs()->ip();
 871   );
 872
 873     {
 874       Lock_guard<typeof(_migration_rq.affinity_lock)>
 875         g(&_migration_rq.affinity_lock);
 876       _migration_rq.inf = info;
 877       _migration_rq.pending = true;
 878     }
 879
 880   unsigned cpu = this->cpu();
 881
 882   if (current_cpu() == cpu)
 883     {
 884       do_migration();
 885       return;
 886     }
 887
 888   migrate_xcpu(cpu);
 889 }
 890
 891
 892 //---------------------------------------------------------------------------
 893 IMPLEMENTATION [fpu && !ux]:
 894
 895 #include "fpu.h"
 896 #include "fpu_alloc.h"
 897 #include "fpu_state.h"
 898
 899 PUBLIC inline NEEDS ["fpu.h"]
 900 void
 901 Thread::spill_fpu()
 902 {
 903   // If we own the FPU, we should never be getting an "FPU unavailable" trap
 904   assert_kdb (Fpu::owner(cpu()) == this);
 905   assert_kdb (state() & Thread_fpu_owner);
 906   assert_kdb (fpu_state());
 907
 908   // Save the FPU state of the previous FPU owner (lazy) if applicable
 909   Fpu::save_state (fpu_state());
 910   state_del_dirty (Thread_fpu_owner);
 911 }
 912
 913
 914 /*
 915  * Handle FPU trap for this context. Assumes disabled interrupts
 916  */
 917 PUBLIC inline NEEDS [Thread::spill_fpu, "fpu_alloc.h","fpu_state.h"]
 918 int
 919 Thread::switchin_fpu(bool alloc_new_fpu = true)
 920 {
 921   unsigned cpu = this->cpu(true);
 922
 923   if (state() & Thread_vcpu_fpu_disabled)
 924     return 0;
 925
 926   // If we own the FPU, we should never be getting an "FPU unavailable" trap
 927   assert_kdb (Fpu::owner(cpu) != this);
 928
 929   // Allocate FPU state slab if we didn't already have one
 930   if (!fpu_state()->state_buffer()
 931       && (EXPECT_FALSE((!alloc_new_fpu
 932                         || (state() & Thread_alien))
 933                        || !Fpu_alloc::alloc_state(_quota, fpu_state()))))
 934     return 0;
 935
 936   // Enable the FPU before accessing it, otherwise recursive trap
 937   Fpu::enable();
 938
 939   // Save the FPU state of the previous FPU owner (lazy) if applicable
 940   if (Fpu::owner(cpu))
 941     nonull_static_cast<Thread*>(Fpu::owner(cpu))->spill_fpu();
 942
 943   // Become FPU owner and restore own FPU state
 944   Fpu::restore_state(fpu_state());
 945
 946   state_add_dirty(Thread_fpu_owner);
 947   Fpu::set_owner(cpu, this);
 948   return 1;
 949 }
 950
 951 PUBLIC inline NEEDS["fpu.h", "fpu_alloc.h"]
 952 void
 953 Thread::transfer_fpu(Thread *to)
 954 {
 955   unsigned cpu = this->cpu();
 956   if (cpu != to->cpu())
 957     return;
 958
 959   if (to->fpu_state()->state_buffer())
 960     Fpu_alloc::free_state(to->fpu_state());
 961
 962   to->fpu_state()->state_buffer(fpu_state()->state_buffer());
 963   fpu_state()->state_buffer(0);
 964
 965   assert (current() == this || current() == to);
 966
 967   Fpu::disable(); // it will be reanabled in switch_fpu
 968
 969   if (EXPECT_FALSE(Fpu::owner(cpu) == to))
 970     {
 971       assert_kdb (to->state() & Thread_fpu_owner);
 972
 973       Fpu::set_owner(cpu, 0);
 974       to->state_del_dirty (Thread_fpu_owner);
 975     }
 976   else if (Fpu::owner(cpu) == this)
 977     {
 978       assert_kdb (state() & Thread_fpu_owner);
 979
 980       state_del_dirty (Thread_fpu_owner);
 981
 982       to->state_add_dirty (Thread_fpu_owner);
 983       Fpu::set_owner(cpu, to);
 984       if (EXPECT_FALSE(current() == to))
 985         Fpu::enable();
 986     }
 987 }
 988
 989 //---------------------------------------------------------------------------
 990 IMPLEMENTATION [!fpu]:
 991
 992 PUBLIC inline
 993 int
 994 Thread::switchin_fpu(bool alloc_new_fpu = true)
 995 {
 996   (void)alloc_new_fpu;
 997   return 0;
 998 }
 999
1000 PUBLIC inline
1001 void
1002 Thread::spill_fpu()
1003 {}
1004
1005 //---------------------------------------------------------------------------
1006 IMPLEMENTATION [!fpu || ux]:
1007
1008 PUBLIC inline
1009 void
1010 Thread::transfer_fpu(Thread *)
1011 {}
1012
1013 //---------------------------------------------------------------------------
1014 IMPLEMENTATION [!log]:
1015
1016 PUBLIC inline
1017 unsigned Thread::sys_ipc_log(Syscall_frame *)
1018 { return 0; }
1019
1020 PUBLIC inline
1021 unsigned Thread::sys_ipc_trace(Syscall_frame *)
1022 { return 0; }
1023
1024 static inline
1025 void Thread::page_fault_log(Address, unsigned, unsigned)
1026 {}
1027
1028 PUBLIC static inline
1029 int Thread::log_page_fault()
1030 { return 0; }
1031
1032 PUBLIC inline
1033 unsigned Thread::sys_fpage_unmap_log(Syscall_frame *)
1034 { return 0; }
1035
1036
1037 // ----------------------------------------------------------------------------
1038 IMPLEMENTATION [!mp]:
1039
1040
1041 PRIVATE inline
1042 unsigned
1043 Thread::migration_helper(Migration_info const *inf)
1044 {
1045   unsigned cpu = inf->cpu;
1046   //  LOG_MSG_3VAL(this, "MGi ", Mword(current()), (current_cpu() << 16) | cpu(), Context::current_sched());
1047   if (_timeout)
1048     _timeout->reset();
1049   ready_dequeue();
1050
1051     {
1052       // Not sure if this can ever happen
1053       Sched_context *csc = Context::current_sched();
1054       if (!csc || csc->context() == this)
1055         Context::set_current_sched(current()->sched());
1056     }
1057
1058   Sched_context *sc = sched_context();
1059   sc->set_prio(inf->prio);
1060   sc->set_quantum(inf->quantum);
1061   sc->replenish();
1062   set_sched(sc);
1063
1064   if (drq_pending())
1065     state_add_dirty(Thread_drq_ready);
1066
1067   set_cpu_of(this, cpu);
1068   return  Drq::No_answer | Drq::Need_resched;
1069 }
1070
1071 PRIVATE inline
1072 void
1073 Thread::migrate_xcpu(unsigned cpu)
1074 {
1075   (void)cpu;
1076   assert_kdb (false);
1077 }
1078
1079
1080 //----------------------------------------------------------------------------
1081 INTERFACE [debug]:
1082
1083 EXTENSION class Thread
1084 {
1085 protected:
1086   struct Migration_log
1087   {
1088     Mword    state;
1089     Address  user_ip;
1090     unsigned src_cpu;
1091     unsigned target_cpu;
1092
1093     static unsigned fmt(Tb_entry *, int, char *)
1094     asm ("__thread_migration_log_fmt");
1095   };
1096 };
1097
1098
1099 // ----------------------------------------------------------------------------
1100 IMPLEMENTATION [mp]:
1101
1102 #include "ipi.h"
1103
1104 IMPLEMENT
1105 void
1106 Thread::handle_remote_requests_irq()
1107 {
1108   assert_kdb (cpu_lock.test());
1109   // printf("CPU[%2u]: > RQ IPI (current=%p)\n", current_cpu(), current());
1110   Ipi::eoi(Ipi::Request);
1111   Context *const c = current();
1112   //LOG_MSG_3VAL(c, "ipi", c->cpu(), (Mword)c, c->drq_pending());
1113   Context *migration_q = 0;
1114   bool resched = _pending_rqq.cpu(c->cpu()).handle_requests(&migration_q);
1115
1116   resched |= Rcu::do_pending_work(c->cpu());
1117
1118   if (migration_q)
1119     static_cast<Thread*>(migration_q)->do_migration();
1120
1121   if ((resched || c->handle_drq()) && !c->schedule_in_progress())
1122     {
1123       //LOG_MSG_3VAL(c, "ipis", 0, 0, 0);
1124       // printf("CPU[%2u]: RQ IPI sched %p\n", current_cpu(), current());
1125       c->schedule();
1126     }
1127   // printf("CPU[%2u]: < RQ IPI (current=%p)\n", current_cpu(), current());
1128 }
1129
1130 IMPLEMENT
1131 void
1132 Thread::handle_global_remote_requests_irq()
1133 {
1134   assert_kdb (cpu_lock.test());
1135   // printf("CPU[%2u]: > RQ IPI (current=%p)\n", current_cpu(), current());
1136   Ipi::eoi(Ipi::Global_request);
1137   Context::handle_global_requests();
1138 }
1139
1140 PRIVATE inline
1141 unsigned
1142 Thread::migration_helper(Migration_info const *inf)
1143 {
1144   // LOG_MSG_3VAL(this, "MGi ", Mword(current()), (current_cpu() << 16) | cpu(), 0);
1145   assert_kdb (cpu() == current_cpu());
1146   assert_kdb (current() != this);
1147   assert_kdb (cpu_lock.test());
1148
1149   if (_timeout)
1150     _timeout->reset();
1151   ready_dequeue();
1152
1153     {
1154       // Not sure if this can ever happen
1155       Sched_context *csc = Context::current_sched();
1156       if (!csc || csc->context() == this)
1157         Context::set_current_sched(current()->sched());
1158     }
1159
1160   unsigned cpu = inf->cpu;
1161
1162     {
1163       Queue &q = _pending_rqq.cpu(current_cpu());
1164       // The queue lock of the current CPU protects the cpu number in
1165       // the thread
1166       Lock_guard<Pending_rqq::Inner_lock> g(q.q_lock());
1167
1168       // potentailly dequeue from our local queue
1169       if (_pending_rq.queued())
1170         check_kdb (q.dequeue(&_pending_rq, Queue_item::Ok));
1171
1172       Sched_context *sc = sched_context();
1173       sc->set_prio(inf->prio);
1174       sc->set_quantum(inf->quantum);
1175       sc->replenish();
1176       set_sched(sc);
1177
1178       if (drq_pending())
1179         state_add_dirty(Thread_drq_ready);
1180
1181       Mem::mp_wmb();
1182
1183       assert_kdb (!in_ready_list());
1184
1185       set_cpu_of(this, cpu);
1186       // now we are migrated away fom current_cpu
1187     }
1188
1189   bool ipi = true;
1190
1191     {
1192       Queue &q = _pending_rqq.cpu(cpu);
1193       Lock_guard<Pending_rqq::Inner_lock> g(q.q_lock());
1194
1195       // migrated meanwhile
1196       if (this->cpu() != cpu || _pending_rq.queued())
1197         return  Drq::No_answer | Drq::Need_resched;
1198
1199       if (q.first())
1200         ipi = false;
1201
1202       q.enqueue(&_pending_rq);
1203     }
1204
1205   if (ipi)
1206     {
1207       //LOG_MSG_3VAL(this, "sipi", current_cpu(), cpu(), (Mword)current());
1208       Ipi::cpu(cpu).send(Ipi::Request);
1209     }
1210
1211   return  Drq::No_answer | Drq::Need_resched;
1212 }
1213
1214 PRIVATE inline
1215 void
1216 Thread::migrate_xcpu(unsigned cpu)
1217 {
1218   bool ipi = true;
1219
1220     {
1221       Queue &q = Context::_pending_rqq.cpu(cpu);
1222       Lock_guard<Pending_rqq::Inner_lock> g(q.q_lock());
1223
1224       // already migrated
1225       if (cpu != this->cpu())
1226         return;
1227
1228       if (q.first())
1229         ipi = false;
1230
1231       if (!_pending_rq.queued())
1232         q.enqueue(&_pending_rq);
1233       else
1234         ipi = false;
1235     }
1236
1237   if (ipi)
1238     Ipi::cpu(cpu).send(Ipi::Request);
1239 }
1240
1241 //----------------------------------------------------------------------------
1242 IMPLEMENTATION [debug]:
1243
1244 IMPLEMENT
1245 unsigned
1246 Thread::Migration_log::fmt(Tb_entry *e, int maxlen, char *buf)
1247 {
1248   Migration_log *l = e->payload<Migration_log>();
1249   return snprintf(buf, maxlen, "migrate from %u to %u (state=%lx user ip=%lx)",
1250       l->src_cpu, l->target_cpu, l->state, l->user_ip);
1251 }
1252