kernel/fiasco/src/kern/thread.cpp

   1 INTERFACE:
   2
   3 #include "l4_types.h"
   4 #include "config.h"
   5 #include "continuation.h"
   6 #include "helping_lock.h"
   7 #include "kobject_iface.h"
   8 #include "mem_layout.h"
   9 #include "member_offs.h"
  10 #include "receiver.h"
  11 #include "ref_obj.h"
  12 #include "sender.h"
  13 #include "space.h"              // Space_index
  14 #include "spin_lock.h"
  15 #include "thread_lock.h"
  16
  17 class Return_frame;
  18 class Syscall_frame;
  19 class Vcpu_state;
  20 class Irq_base;
  21
  22 typedef Context_ptr_base<Thread> Thread_ptr;
  23
  24
  25 /** A thread.  This class is the driver class for most kernel functionality.
  26  */
  27 class Thread :
  28   public Receiver,
  29   public Sender,
  30   public Kobject_iface
  31 {
  32   MEMBER_OFFSET();
  33   friend class Jdb;
  34   friend class Jdb_bt;
  35   friend class Jdb_tcb;
  36   friend class Jdb_thread;
  37   friend class Jdb_thread_list;
  38   friend class Jdb_list_threads;
  39   friend class Jdb_list_timeouts;
  40   friend class Jdb_tbuf_show;
  41
  42 public:
  43   enum Context_mode_kernel { Kernel = 0 };
  44   enum Operation
  45   {
  46     Opcode_mask = 0xffff,
  47     Op_control = 0,
  48     Op_ex_regs = 1,
  49     Op_switch  = 2,
  50     Op_stats   = 3,
  51     Op_vcpu_resume = 4,
  52     Op_register_del_irq = 5,
  53     Op_modify_senders = 6,
  54     Op_gdt_x86 = 0x10,
  55   };
  56
  57   enum Control_flags
  58   {
  59     Ctl_set_pager       = 0x0010000,
  60     Ctl_set_scheduler   = 0x0020000,
  61     Ctl_set_mcp         = 0x0040000,
  62     Ctl_set_prio        = 0x0080000,
  63     Ctl_set_quantum     = 0x0100000,
  64     Ctl_bind_task       = 0x0200000,
  65     Ctl_alien_thread    = 0x0400000,
  66     Ctl_ux_native       = 0x0800000,
  67     Ctl_set_exc_handler = 0x1000000,
  68     Ctl_vcpu_enabled    = 0x2000000,
  69   };
  70
  71   enum Ex_regs_flags
  72   {
  73     Exr_cancel            = 0x10000,
  74     Exr_trigger_exception = 0x20000,
  75   };
  76
  77   struct Remote_syscall
  78   {
  79     Thread *thread;
  80     L4_msg_tag result;
  81   };
  82
  83
  84   class Dbg_stack
  85   {
  86   public:
  87     enum { Stack_size = Config::PAGE_SIZE };
  88     void *stack_top;
  89     Dbg_stack();
  90   };
  91
  92   static Per_cpu<Dbg_stack> dbg_stack;
  93
  94 public:
  95
  96   typedef void (Utcb_copy_func)(Thread *sender, Thread *receiver);
  97
  98   /**
  99    * Constructor.
 100    *
 101    * @param task the task the thread should reside in.
 102    * @param id user-visible thread ID of the sender.
 103    * @param init_prio initial priority.
 104    * @param mcp maximum controlled priority.
 105    *
 106    * @post state() != Thread_invalid.
 107    */
 108   Thread();
 109
 110
 111   int handle_page_fault (Address pfa, Mword error, Mword pc,
 112       Return_frame *regs);
 113
 114 private:
 115
 116   struct Migration_helper_info
 117   {
 118     Migration_info inf;
 119     Thread *victim;
 120   };
 121
 122   Thread(const Thread&);        ///< Default copy constructor is undefined
 123   void *operator new(size_t);   ///< Default new operator undefined
 124
 125   bool handle_sigma0_page_fault (Address pfa);
 126
 127   /**
 128    * Return to user.
 129    *
 130    * This function is the default routine run if a newly
 131    * initialized context is being switch_exec()'ed.
 132    */
 133   static void user_invoke();
 134
 135 public:
 136   static bool pagein_tcb_request(Return_frame *regs);
 137
 138   inline Mword user_ip() const;
 139   inline void user_ip(Mword);
 140
 141   inline Mword user_sp() const;
 142   inline void user_sp(Mword);
 143
 144   inline Mword user_flags() const;
 145
 146   /** nesting level in debugger (always critical) if >1 */
 147   static Per_cpu<unsigned long> nested_trap_recover;
 148   static void handle_remote_requests_irq() asm ("handle_remote_cpu_requests");
 149   static void handle_global_remote_requests_irq() asm ("ipi_remote_call");
 150 protected:
 151   // implementation details follow...
 152
 153   explicit Thread(Context_mode_kernel);
 154
 155   // DATA
 156
 157   // Another critical TCB cache line:
 158   Thread_lock  _thread_lock;
 159
 160   // More ipc state
 161   Thread_ptr _pager;
 162   Thread_ptr _exc_handler;
 163
 164 protected:
 165   Ram_quota *_quota;
 166   Irq_base *_del_observer;
 167
 168   // debugging stuff
 169   unsigned _magic;
 170   static const unsigned magic = 0xf001c001;
 171 };
 172
 173
 174 IMPLEMENTATION:
 175
 176 #include <cassert>
 177 #include <cstdlib>              // panic()
 178 #include <cstring>
 179 #include "atomic.h"
 180 #include "entry_frame.h"
 181 #include "fpu_alloc.h"
 182 #include "globals.h"
 183 #include "kdb_ke.h"
 184 #include "kmem_alloc.h"
 185 #include "logdefs.h"
 186 #include "map_util.h"
 187 #include "ram_quota.h"
 188 #include "sched_context.h"
 189 #include "space.h"
 190 #include "std_macros.h"
 191 #include "task.h"
 192 #include "thread_state.h"
 193 #include "timeout.h"
 194
 195
 196 Per_cpu<unsigned long> DEFINE_PER_CPU Thread::nested_trap_recover;
 197
 198
 199 IMPLEMENT
 200 Thread::Dbg_stack::Dbg_stack()
 201 {
 202   stack_top = Kmem_alloc::allocator()->unaligned_alloc(Stack_size);
 203   if (stack_top)
 204     stack_top = (char *)stack_top + Stack_size;
 205   //printf("JDB STACK start= %p - %p\n", (char *)stack_top - Stack_size, (char *)stack_top);
 206 }
 207
 208
 209 PUBLIC inline NEEDS[Thread::thread_lock]
 210 void
 211 Thread::kill_lock()
 212 {
 213   thread_lock()->lock();
 214 }
 215
 216
 217 PUBLIC inline
 218 void *
 219 Thread::operator new(size_t, Ram_quota *q) throw ()
 220 {
 221   void *t = Mapped_allocator::allocator()->q_unaligned_alloc(q, Config::thread_block_size);
 222   if (t)
 223     {
 224       memset(t, 0, sizeof(Thread));
 225       reinterpret_cast<Thread*>(t)->_quota = q;
 226     }
 227   return t;
 228 }
 229
 230 /** Class-specific allocator.
 231     This allocator ensures that threads are allocated at a fixed virtual
 232     address computed from their thread ID.
 233     @param id thread ID
 234     @return address of new thread control block
 235  */
 236 PRIVATE inline
 237 void *
 238 Thread::operator new(size_t, Thread *t) throw ()
 239 {
 240   // Allocate TCB in TCB space.  Actually, do not allocate anything,
 241   // just return the address.  Allocation happens on the fly in
 242   // Thread::handle_page_fault().
 243   return t;
 244 }
 245
 246
 247 PUBLIC inline NEEDS["space.h"]
 248 bool
 249 Thread::bind(Space *t, void *_utcb)
 250 {
 251   // _utcb == 0 for all kernel threads
 252   assert_kdb (!_utcb || t->is_utcb_valid(_utcb));
 253
 254     {
 255       Lock_guard<Spin_lock> guard(&_space);
 256       if (_space.get_unused())
 257         return false;
 258
 259       _space.set_unused(t);
 260       space()->inc_ref();
 261     }
 262
 263   utcb(t->kernel_utcb(_utcb));
 264   local_id(Address(_utcb));
 265   _utcb_handler = 0;
 266
 267   return true;
 268 }
 269
 270
 271 PUBLIC inline NEEDS["kdb_ke.h", "cpu_lock.h", "space.h"]
 272 bool
 273 Thread::unbind()
 274 {
 275   Space *old;
 276
 277     {
 278       Lock_guard<Spin_lock> guard(&_space);
 279
 280       if (!_space.get_unused())
 281         return true;
 282
 283       old = _space.get_unused();
 284       _space.set_unused(0);
 285
 286       Mem_space *oms = old->mem_space();
 287
 288       if (old->dec_ref())
 289         old = 0;
 290
 291       // switch to a safe page table
 292       if (Mem_space::current_mem_space(current_cpu()) == oms)
 293         Mem_space::kernel_space()->switchin_context(oms);
 294     }
 295
 296   if (old)
 297     {
 298       current()->rcu_wait();
 299       delete old;
 300     }
 301
 302   return true;
 303 }
 304
 305 /** Cut-down version of Thread constructor; only for kernel threads
 306     Do only what's necessary to get a kernel thread started --
 307     skip all fancy stuff, no locking is necessary.
 308     @param task the address space
 309     @param id user-visible thread ID of the sender
 310  */
 311 IMPLEMENT inline
 312 Thread::Thread(Context_mode_kernel)
 313   : Receiver(&_thread_lock), Sender(), _del_observer(0), _magic(magic)
 314 {
 315   *reinterpret_cast<void(**)()>(--_kernel_sp) = user_invoke;
 316
 317   inc_ref();
 318
 319   if (Config::stack_depth)
 320     std::memset((char*)this + sizeof(Thread), '5',
 321                 Config::thread_block_size-sizeof(Thread)-64);
 322 }
 323
 324
 325 /** Destructor.  Reestablish the Context constructor's precondition.
 326     @pre current() == thread_lock()->lock_owner()
 327          && state() == Thread_dead
 328     @pre lock_cnt() == 0
 329     @post (_kernel_sp == 0)  &&  (* (stack end) == 0)  &&  !exists()
 330  */
 331 PUBLIC virtual
 332 Thread::~Thread()               // To be called in locked state.
 333 {
 334
 335   unsigned long *init_sp = reinterpret_cast<unsigned long*>
 336     (reinterpret_cast<unsigned long>(this) + size - sizeof(Entry_frame));
 337
 338
 339   _kernel_sp = 0;
 340   *--init_sp = 0;
 341   Fpu_alloc::free_state(fpu_state());
 342   state_change(0, Thread_invalid);
 343 }
 344
 345
 346 // IPC-gate deletion stuff ------------------------------------
 347
 348 PUBLIC inline
 349 void
 350 Thread::ipc_gate_deleted(Mword id)
 351 {
 352   (void) id;
 353   Lock_guard<Cpu_lock> g(&cpu_lock);
 354   if (_del_observer)
 355     _del_observer->hit();
 356 }
 357
 358 class Del_irq_pin : public Irq_pin_dummy
 359 {
 360 };
 361
 362 PUBLIC inline
 363 Del_irq_pin::Del_irq_pin(Thread *o)
 364 { payload()[0] = (Address)o; }
 365
 366 PUBLIC inline
 367 Thread *
 368 Del_irq_pin::thread() const
 369 { return (Thread*)payload()[0]; }
 370
 371 PUBLIC inline
 372 void
 373 Del_irq_pin::unbind_irq()
 374 {
 375   thread()->remove_delete_irq();
 376 }
 377
 378 PUBLIC inline
 379 Del_irq_pin::~Del_irq_pin()
 380 {
 381   unbind_irq();
 382 }
 383
 384 PUBLIC
 385 void
 386 Thread::register_delete_irq(Irq_base *irq)
 387 {
 388   irq->pin()->unbind_irq();
 389   irq->pin()->replace<Del_irq_pin>(this);
 390   _del_observer = irq;
 391 }
 392
 393 PUBLIC
 394 void
 395 Thread::remove_delete_irq()
 396 {
 397   if (!_del_observer)
 398     return;
 399
 400   Irq_base *tmp = _del_observer;
 401   _del_observer = 0;
 402   tmp->pin()->unbind_irq();
 403 }
 404
 405 // end of: IPC-gate deletion stuff -------------------------------
 406
 407
 408 /** Lookup function: Find Thread instance that owns a given Context.
 409     @param c a context
 410     @return the thread that owns the context
 411  */
 412 PUBLIC static inline
 413 Thread*
 414 Thread::lookup (Context* c)
 415 {
 416   return reinterpret_cast<Thread*>(c);
 417 }
 418
 419 PUBLIC static inline
 420 Thread const *
 421 Thread::lookup (Context const * c)
 422 {
 423   return reinterpret_cast<Thread const *>(c);
 424 }
 425
 426 /** Currently executing thread.
 427     @return currently executing thread.
 428  */
 429 inline
 430 Thread*
 431 current_thread()
 432 {
 433   return Thread::lookup(current());
 434 }
 435
 436 PUBLIC inline
 437 bool
 438 Thread::exception_triggered() const
 439 { return _exc_cont.valid(); }
 440
 441 //
 442 // state requests/manipulation
 443 //
 444
 445
 446 /** Thread lock.
 447     Overwrite Context's version of thread_lock() with a semantically
 448     equivalent, but more efficient version.
 449     @return lock used to synchronize accesses to the thread.
 450  */
 451 PUBLIC inline
 452 Thread_lock *
 453 Thread::thread_lock()
 454 {
 455   return &_thread_lock;
 456 }
 457
 458
 459 PUBLIC inline NEEDS ["config.h", "timeout.h"]
 460 void
 461 Thread::handle_timer_interrupt()
 462 {
 463   unsigned _cpu = cpu(true);
 464   // XXX: This assumes periodic timers (i.e. bogus in one-shot mode)
 465   if (!Config::fine_grained_cputime)
 466     consume_time(Config::scheduler_granularity);
 467
 468   bool resched = Rcu::do_pending_work(_cpu);
 469
 470   // Check if we need to reschedule due to timeouts or wakeups
 471   if ((Timeout_q::timeout_queue.cpu(_cpu).do_timeouts() || resched)
 472       && !schedule_in_progress())
 473     {
 474       schedule();
 475       assert (timeslice_timeout.cpu(cpu(true))->is_set());      // Coma check
 476     }
 477 }
 478
 479
 480 PUBLIC
 481 void
 482 Thread::halt()
 483 {
 484   // Cancel must be cleared on all kernel entry paths. See slowtraps for
 485   // why we delay doing it until here.
 486   state_del (Thread_cancel);
 487
 488   // we haven't been re-initialized (cancel was not set) -- so sleep
 489   if (state_change_safely (~Thread_ready, Thread_cancel | Thread_dead))
 490     while (! (state() & Thread_ready))
 491       schedule();
 492 }
 493
 494 PUBLIC static
 495 void
 496 Thread::halt_current ()
 497 {
 498   for (;;)
 499     {
 500       current_thread()->halt();
 501       kdb_ke("Thread not halted");
 502     }
 503 }
 504
 505 PRIVATE static inline
 506 void
 507 Thread::user_invoke_generic()
 508 {
 509   Context *const c = current();
 510   assert_kdb (c->state() & Thread_ready_mask);
 511
 512   if (c->handle_drq() && !c->schedule_in_progress())
 513     c->schedule();
 514
 515   // release CPU lock explicitly, because
 516   // * the context that switched to us holds the CPU lock
 517   // * we run on a newly-created stack without a CPU lock guard
 518   cpu_lock.clear();
 519 }
 520
 521
 522 PRIVATE static void
 523 Thread::leave_and_kill_myself()
 524 {
 525   current_thread()->do_kill();
 526 #ifdef CONFIG_JDB
 527   WARN("dead thread scheduled: %lx\n", current_thread()->kobject()->dbg_id());
 528 #endif
 529   kdb_ke("DEAD SCHED");
 530 }
 531
 532 PUBLIC static
 533 unsigned
 534 Thread::handle_kill_helper(Drq *src, Context *, void *)
 535 {
 536   delete nonull_static_cast<Thread*>(src->context());
 537   return Drq::No_answer | Drq::Need_resched;
 538 }
 539
 540
 541
 542 PRIVATE
 543 bool
 544 Thread::do_kill()
 545 {
 546   Lock_guard<Thread_lock> guard(thread_lock());
 547
 548   if (state() == Thread_invalid)
 549     return false;
 550
 551
 552   unset_utcb_ptr();
 553
 554
 555   //
 556   // Kill this thread
 557   //
 558
 559   // But first prevent it from being woken up by asynchronous events
 560
 561   {
 562     Lock_guard <Cpu_lock> guard (&cpu_lock);
 563
 564     // if IPC timeout active, reset it
 565     if (_timeout)
 566       _timeout->reset();
 567
 568     // Switch to time-sharing mode
 569     set_mode (Sched_mode (0));
 570
 571     // Switch to time-sharing scheduling context
 572     if (sched() != sched_context())
 573       switch_sched(sched_context());
 574
 575     if (current_sched()->context() == this)
 576       set_current_sched(current()->sched());
 577   }
 578
 579   // possibly dequeue from a wait queue
 580   wait_queue_kill();
 581
 582   // if other threads want to send me IPC messages, abort these
 583   // operations
 584   {
 585     Lock_guard <Cpu_lock> guard (&cpu_lock);
 586     while (Sender *s = Sender::cast(sender_list()->head()))
 587       {
 588         s->ipc_receiver_aborted();
 589         Proc::preemption_point();
 590       }
 591   }
 592
 593   // if engaged in IPC operation, stop it
 594   if (receiver())
 595     sender_dequeue (receiver()->sender_list());
 596
 597   Context::do_kill();
 598
 599   vcpu_update_state();
 600
 601   unbind();
 602   vcpu_set_user_space(0);
 603
 604   cpu_lock.lock();
 605
 606   state_change_dirty (0, Thread_dead);
 607
 608   // dequeue from system queues
 609   ready_dequeue();
 610
 611   if (_del_observer)
 612     {
 613       _del_observer->pin()->unbind_irq();
 614       _del_observer = 0;
 615     }
 616
 617   if (dec_ref())
 618     while (1)
 619       {
 620         state_del_dirty(Thread_ready_mask);
 621         schedule();
 622         WARN("woken up dead thread %lx\n", kobject()->dbg_id());
 623         kdb_ke("X");
 624       }
 625
 626   rcu_wait();
 627
 628   state_del_dirty(Thread_ready_mask);
 629
 630   ready_dequeue();
 631
 632   kernel_context_drq(handle_kill_helper, 0);
 633   kdb_ke("Im dead");
 634   return true;
 635 }
 636
 637 PRIVATE static
 638 unsigned
 639 Thread::handle_remote_kill(Drq *, Context *self, void *)
 640 {
 641   Thread *c = nonull_static_cast<Thread*>(self);
 642   c->state_add_dirty(Thread_cancel | Thread_ready);
 643   c->_exc_cont.restore(c->regs());
 644   c->do_trigger_exception(c->regs(), (void*)&Thread::leave_and_kill_myself);
 645   return 0;
 646 }
 647
 648
 649 PROTECTED
 650 bool
 651 Thread::kill()
 652 {
 653   Lock_guard<Cpu_lock> guard(&cpu_lock);
 654   inc_ref();
 655
 656
 657   if (cpu() == current_cpu())
 658     {
 659       state_add_dirty(Thread_cancel | Thread_ready);
 660       sched()->deblock(cpu());
 661       _exc_cont.restore(regs()); // overwrite an already triggered exception
 662       do_trigger_exception(regs(), (void*)&Thread::leave_and_kill_myself);
 663 //          current()->switch_exec (this, Helping);
 664       return true;
 665     }
 666
 667   drq(Thread::handle_remote_kill, 0, 0, Drq::Any_ctxt);
 668
 669   return true;
 670 #if 0
 671     drq(Thread::handle_migration, reinterpret_cast<void*>(current_cpu()));
 672
 673   assert_kdb(cpu() == current_cpu());
 674
 675   return do_kill();
 676 #endif
 677 }
 678
 679
 680 PUBLIC
 681 void
 682 Thread::set_sched_params(unsigned prio, Unsigned64 quantum)
 683 {
 684   Sched_context *sc = sched_context();
 685   bool const change = prio != sc->prio()
 686                    || quantum != sc->quantum();
 687   bool const ready_queued = in_ready_list();
 688
 689   if (!change && (ready_queued || this == current()))
 690     return;
 691
 692   ready_dequeue();
 693
 694   sc->set_prio(prio);
 695   sc->set_quantum(quantum);
 696   sc->replenish();
 697
 698   if (sc == current_sched())
 699     set_current_sched(sc);
 700
 701   if (state() & Thread_ready_mask)
 702     {
 703       if (this != current())
 704         ready_enqueue();
 705       else
 706         schedule();
 707     }
 708 }
 709
 710 PUBLIC
 711 long
 712 Thread::control(Thread_ptr const &pager, Thread_ptr const &exc_handler,
 713                 Space *task, void *user_utcb,
 714                 bool utcb_vcpu_flag = false, bool utcb_vcpu_val = false)
 715 {
 716   bool new_vcpu_state = state() & Thread_vcpu_enabled;
 717   if (utcb_vcpu_flag)
 718     new_vcpu_state = utcb_vcpu_val;
 719
 720   if (task)
 721     {
 722       if (EXPECT_FALSE(!task->is_utcb_valid(user_utcb, 1 + new_vcpu_state)))
 723         return -L4_err::EInval;
 724
 725       if (EXPECT_FALSE(!bind(task, user_utcb)))
 726         return -L4_err::EInval; // unbind first !!
 727
 728       if (new_vcpu_state)
 729         vcpu_state(utcb() + 1);
 730     }
 731
 732   if (new_vcpu_state)
 733     {
 734       if (space())
 735         {
 736           if (!space()->is_utcb_valid((void *)local_id(), 2))
 737             return -L4_err::EInval;
 738           vcpu_state(utcb() + 1);
 739         }
 740
 741       state_add_dirty(Thread_vcpu_enabled);
 742     }
 743   else
 744     // we're not clearing the vcpu_state pointer, it's not used if vcpu mode
 745     // is off
 746     state_del_dirty(Thread_vcpu_enabled);
 747
 748   if (pager.is_valid())
 749     _pager = pager;
 750
 751   if (exc_handler.is_valid())
 752     _exc_handler = exc_handler;
 753
 754   return 0;
 755 }
 756
 757
 758 /** Clears the utcb pointer of the Thread
 759  *  Reason: To avoid a stale pointer after unmapping and deallocating
 760  *  the UTCB. Without this the Thread_lock::clear will access the UTCB
 761  *  after the unmapping the UTCB -> POOFFF.
 762  */
 763 PUBLIC inline
 764 void
 765 Thread::unset_utcb_ptr()
 766 {
 767   utcb(0);
 768   local_id(0);
 769 }
 770
 771
 772 PRIVATE static inline
 773 bool FIASCO_WARN_RESULT
 774 Thread::copy_utcb_to_utcb(L4_msg_tag const &tag, Thread *snd, Thread *rcv,
 775                           unsigned char rights)
 776 {
 777   assert (cpu_lock.test());
 778
 779   Utcb *snd_utcb = snd->access_utcb();
 780   Utcb *rcv_utcb = rcv->access_utcb();
 781   Mword s = tag.words();
 782   Mword r = Utcb::Max_words;
 783
 784   Mem::memcpy_mwords (rcv_utcb->values, snd_utcb->values, r < s ? r : s);
 785
 786   bool success = true;
 787   if (tag.items())
 788     success = transfer_msg_items(tag, snd, snd_utcb, rcv, rcv_utcb, rights);
 789
 790   if (tag.transfer_fpu() && rcv_utcb->inherit_fpu() && (rights & L4_fpage::W))
 791     snd->transfer_fpu(rcv);
 792
 793   return success;
 794 }
 795
 796
 797 PUBLIC inline NEEDS[Thread::copy_utcb_to_ts, Thread::copy_utcb_to_utcb,
 798                     Thread::copy_ts_to_utcb]
 799 bool FIASCO_WARN_RESULT
 800 Thread::copy_utcb_to(L4_msg_tag const &tag, Thread* receiver,
 801                      unsigned char rights)
 802 {
 803   // we cannot copy trap state to trap state!
 804   assert_kdb (!this->_utcb_handler || !receiver->_utcb_handler);
 805   if (EXPECT_FALSE(this->_utcb_handler != 0))
 806     return copy_ts_to_utcb(tag, this, receiver, rights);
 807   else if (EXPECT_FALSE(receiver->_utcb_handler != 0))
 808     return copy_utcb_to_ts(tag, this, receiver, rights);
 809   else
 810     return copy_utcb_to_utcb(tag, this, receiver, rights);
 811 }
 812
 813
 814 PUBLIC static inline
 815 bool
 816 Thread::is_tcb_address(Address a)
 817 {
 818   a &= ~(Config::thread_block_size - 1);
 819   return reinterpret_cast<Thread *>(a)->_magic == magic;
 820 }
 821
 822 PUBLIC static inline
 823 void
 824 Thread::assert_irq_entry()
 825 {
 826   assert_kdb(current_thread()->schedule_in_progress()
 827              || current_thread()->state() & (Thread_ready_mask | Thread_drq_wait | Thread_waiting));
 828 }
 829
 830
 831
 832 // ---------------------------------------------------------------------------
 833
 834 PUBLIC inline
 835 bool
 836 Thread::check_sys_ipc(unsigned flags, Thread **partner, Thread **sender,
 837                       bool *have_recv) const
 838 {
 839   if (flags & L4_obj_ref::Ipc_recv)
 840     {
 841       *sender = flags & L4_obj_ref::Ipc_open_wait ? 0 : const_cast<Thread*>(this);
 842       *have_recv = true;
 843     }
 844
 845   if (flags & L4_obj_ref::Ipc_send)
 846     *partner = const_cast<Thread*>(this);
 847
 848   // FIXME: shall be removed flags == 0 is no-op
 849   if (!flags)
 850     {
 851       *sender = const_cast<Thread*>(this);
 852       *partner = const_cast<Thread*>(this);
 853       *have_recv = true;
 854     }
 855
 856   return *have_recv || ((flags & L4_obj_ref::Ipc_send) && *partner);
 857 }
 858
 859 PUBLIC static
 860 unsigned
 861 Thread::handle_migration_helper(Drq *, Context *, void *p)
 862 {
 863   Migration_helper_info const *inf = (Migration_helper_info const *)p;
 864   return inf->victim->migration_helper(&inf->inf);
 865 }
 866
 867
 868 PRIVATE
 869 void
 870 Thread::do_migration()
 871 {
 872   assert_kdb(cpu_lock.test());
 873   assert_kdb(current_cpu() == cpu(true));
 874
 875   Migration_helper_info inf;
 876
 877     {
 878       Lock_guard<Spin_lock> g(affinity_lock());
 879       inf.inf = _migration_rq.inf;
 880       _migration_rq.pending = false;
 881       _migration_rq.in_progress = true;
 882     }
 883
 884   unsigned on_cpu = cpu();
 885
 886   if (inf.inf.cpu == ~0U)
 887     {
 888       state_add_dirty(Thread_suspended);
 889       set_sched_params(0, 0);
 890       _migration_rq.in_progress = false;
 891       return;
 892     }
 893
 894   state_del_dirty(Thread_suspended);
 895
 896   if (inf.inf.cpu == on_cpu)
 897     {
 898       // stay here
 899       set_sched_params(inf.inf.prio, inf.inf.quantum);
 900       _migration_rq.in_progress = false;
 901       return;
 902     }
 903
 904   // spill FPU state into memory before migration
 905   if (state() & Thread_fpu_owner)
 906     {
 907       if (current() != this)
 908         Fpu::enable();
 909
 910       spill_fpu();
 911       Fpu::set_owner(on_cpu, 0);
 912       Fpu::disable();
 913     }
 914
 915
 916   // if we are in the middle of the scheduler, leave it now
 917   if (schedule_in_progress() == this)
 918     reset_schedule_in_progress();
 919
 920   inf.victim = this;
 921
 922   if (current() == this && Config::Max_num_cpus > 1)
 923     kernel_context_drq(handle_migration_helper, &inf);
 924   else
 925     migration_helper(&inf.inf);
 926 }
 927
 928 PUBLIC
 929 void
 930 Thread::initiate_migration()
 931 { do_migration(); }
 932
 933 PUBLIC
 934 void
 935 Thread::finish_migration()
 936 { enqueue_timeout_again(); }
 937
 938
 939 PUBLIC
 940 void
 941 Thread::migrate(Migration_info const &info)
 942 {
 943   assert_kdb (cpu_lock.test());
 944
 945   LOG_TRACE("Thread migration", "mig", this, __thread_migration_log_fmt,
 946       Migration_log *l = tbe->payload<Migration_log>();
 947       l->state = state();
 948       l->src_cpu = cpu();
 949       l->target_cpu = info.cpu;
 950       l->user_ip = regs()->ip();
 951   );
 952
 953     {
 954       Lock_guard<Spin_lock> g(affinity_lock());
 955       _migration_rq.inf = info;
 956       _migration_rq.pending = true;
 957     }
 958
 959   unsigned cpu = this->cpu();
 960
 961   if (current_cpu() == cpu)
 962     {
 963       do_migration();
 964       return;
 965     }
 966
 967   migrate_xcpu(cpu);
 968 }
 969
 970
 971 //---------------------------------------------------------------------------
 972 IMPLEMENTATION [fpu && !ux]:
 973
 974 #include "fpu.h"
 975 #include "fpu_alloc.h"
 976 #include "fpu_state.h"
 977
 978 PUBLIC inline NEEDS ["fpu.h"]
 979 void
 980 Thread::spill_fpu()
 981 {
 982   // If we own the FPU, we should never be getting an "FPU unavailable" trap
 983   assert_kdb (Fpu::owner(cpu()) == this);
 984   assert_kdb (state() & Thread_fpu_owner);
 985   assert_kdb (fpu_state());
 986
 987   // Save the FPU state of the previous FPU owner (lazy) if applicable
 988   Fpu::save_state (fpu_state());
 989   state_del_dirty (Thread_fpu_owner);
 990 }
 991
 992
 993 /*
 994  * Handle FPU trap for this context. Assumes disabled interrupts
 995  */
 996 PUBLIC inline NEEDS [Thread::spill_fpu, "fpu_alloc.h","fpu_state.h"]
 997 int
 998 Thread::switchin_fpu(bool alloc_new_fpu = true)
 999 {
1000   unsigned cpu = this->cpu(true);
1001
1002   if (state() & Thread_vcpu_fpu_disabled)
1003     return 0;
1004
1005   // If we own the FPU, we should never be getting an "FPU unavailable" trap
1006   assert_kdb (Fpu::owner(cpu) != this);
1007
1008   // Allocate FPU state slab if we didn't already have one
1009   if (!fpu_state()->state_buffer()
1010       && (EXPECT_FALSE((!alloc_new_fpu
1011                         || (state() & Thread_alien))
1012                        || !Fpu_alloc::alloc_state (_quota, fpu_state()))))
1013     return 0;
1014
1015   // Enable the FPU before accessing it, otherwise recursive trap
1016   Fpu::enable();
1017
1018   // Save the FPU state of the previous FPU owner (lazy) if applicable
1019   if (Fpu::owner(cpu))
1020     nonull_static_cast<Thread*>(Fpu::owner(cpu))->spill_fpu();
1021
1022   // Become FPU owner and restore own FPU state
1023   Fpu::restore_state (fpu_state());
1024
1025   state_add_dirty (Thread_fpu_owner);
1026   Fpu::set_owner (cpu, this);
1027   return 1;
1028 }
1029
1030 PUBLIC inline NEEDS["fpu.h", "fpu_alloc.h"]
1031 void
1032 Thread::transfer_fpu(Thread *to)
1033 {
1034   unsigned cpu = this->cpu();
1035   if (cpu != to->cpu())
1036     return;
1037
1038   if (to->fpu_state()->state_buffer())
1039     Fpu_alloc::free_state(to->fpu_state());
1040
1041   to->fpu_state()->state_buffer(fpu_state()->state_buffer());
1042   fpu_state()->state_buffer(0);
1043
1044   assert (current() == this || current() == to);
1045
1046   Fpu::disable(); // it will be reanabled in switch_fpu
1047
1048   if (EXPECT_FALSE(Fpu::owner(cpu) == to))
1049     {
1050       assert_kdb (to->state() & Thread_fpu_owner);
1051
1052       Fpu::set_owner(cpu, 0);
1053       to->state_del_dirty (Thread_fpu_owner);
1054     }
1055   else if (Fpu::owner(cpu) == this)
1056     {
1057       assert_kdb (state() & Thread_fpu_owner);
1058
1059       state_del_dirty (Thread_fpu_owner);
1060
1061       to->state_add_dirty (Thread_fpu_owner);
1062       Fpu::set_owner(cpu, to);
1063       if (EXPECT_FALSE(current() == to))
1064         Fpu::enable();
1065     }
1066 }
1067
1068 //---------------------------------------------------------------------------
1069 IMPLEMENTATION [!fpu]:
1070
1071 PUBLIC inline
1072 int
1073 Thread::switchin_fpu(bool alloc_new_fpu = true)
1074 { (void)alloc_new_fpu;
1075   return 0;
1076 }
1077
1078 PUBLIC inline
1079 void
1080 Thread::spill_fpu()
1081 {}
1082
1083 //---------------------------------------------------------------------------
1084 IMPLEMENTATION [!fpu || ux]:
1085
1086 PUBLIC inline
1087 void
1088 Thread::transfer_fpu(Thread *)
1089 {}
1090
1091 //---------------------------------------------------------------------------
1092 IMPLEMENTATION [!log]:
1093
1094 PUBLIC inline
1095 unsigned Thread::sys_ipc_log(Syscall_frame *)
1096 { return 0; }
1097
1098 PUBLIC inline
1099 unsigned Thread::sys_ipc_trace(Syscall_frame *)
1100 { return 0; }
1101
1102 static inline
1103 void Thread::page_fault_log(Address, unsigned, unsigned)
1104 {}
1105
1106 PUBLIC static inline
1107 int Thread::log_page_fault()
1108 { return 0; }
1109
1110 PUBLIC inline
1111 unsigned Thread::sys_fpage_unmap_log(Syscall_frame *)
1112 { return 0; }
1113
1114 //---------------------------------------------------------------------------
1115 IMPLEMENTATION [!io]:
1116
1117 PUBLIC inline
1118 bool
1119 Thread::has_privileged_iopl()
1120 {
1121   return false;
1122 }
1123
1124
1125 // ----------------------------------------------------------------------------
1126 IMPLEMENTATION [!mp]:
1127
1128
1129 PRIVATE inline
1130 unsigned
1131 Thread::migration_helper(Migration_info const *inf)
1132 {
1133   unsigned cpu = inf->cpu;
1134   //  LOG_MSG_3VAL(this, "MGi ", Mword(current()), (current_cpu() << 16) | cpu(), Context::current_sched());
1135   if (_timeout)
1136     _timeout->reset();
1137   ready_dequeue();
1138
1139     {
1140       // Not sure if this can ever happen
1141       Sched_context *csc = Context::current_sched();
1142       if (!csc || csc->context() == this)
1143         Context::set_current_sched(current()->sched());
1144     }
1145
1146   Sched_context *sc = sched_context();
1147   sc->set_prio(inf->prio);
1148   sc->set_quantum(inf->quantum);
1149   sc->replenish();
1150   set_sched(sc);
1151
1152   if (drq_pending())
1153     state_add_dirty(Thread_drq_ready);
1154
1155   set_cpu_of(this, cpu);
1156   return  Drq::No_answer | Drq::Need_resched;
1157 }
1158
1159 PRIVATE inline
1160 void
1161 Thread::migrate_xcpu(unsigned cpu)
1162 {
1163   (void)cpu;
1164   assert_kdb (false);
1165 }
1166
1167
1168 //----------------------------------------------------------------------------
1169 INTERFACE [debug]:
1170
1171 EXTENSION class Thread
1172 {
1173 protected:
1174   struct Migration_log
1175   {
1176     Mword    state;
1177     Address  user_ip;
1178     unsigned src_cpu;
1179     unsigned target_cpu;
1180
1181     static unsigned fmt(Tb_entry *, int, char *)
1182     asm ("__thread_migration_log_fmt");
1183   };
1184 };
1185
1186
1187 // ----------------------------------------------------------------------------
1188 IMPLEMENTATION [mp]:
1189
1190 #include "ipi.h"
1191
1192 IMPLEMENT
1193 void
1194 Thread::handle_remote_requests_irq()
1195 { assert_kdb (cpu_lock.test());
1196   // printf("CPU[%2u]: > RQ IPI (current=%p)\n", current_cpu(), current());
1197   Ipi::eoi(Ipi::Request);
1198   Context *const c = current();
1199   //LOG_MSG_3VAL(c, "ipi", c->cpu(), (Mword)c, c->drq_pending());
1200   Context *migration_q = 0;
1201   bool resched = _pending_rqq.cpu(c->cpu()).handle_requests(&migration_q);
1202
1203   resched |= Rcu::do_pending_work(c->cpu());
1204
1205   if (migration_q)
1206     static_cast<Thread*>(migration_q)->do_migration();
1207
1208   if ((resched || c->handle_drq()) && !c->schedule_in_progress())
1209     {
1210       //LOG_MSG_3VAL(c, "ipis", 0, 0, 0);
1211       // printf("CPU[%2u]: RQ IPI sched %p\n", current_cpu(), current());
1212       c->schedule();
1213     }
1214   // printf("CPU[%2u]: < RQ IPI (current=%p)\n", current_cpu(), current());
1215 }
1216
1217 IMPLEMENT
1218 void
1219 Thread::handle_global_remote_requests_irq()
1220 { assert_kdb (cpu_lock.test());
1221   // printf("CPU[%2u]: > RQ IPI (current=%p)\n", current_cpu(), current());
1222   Ipi::eoi(Ipi::Global_request);
1223   Context::handle_global_requests();
1224 }
1225
1226 PRIVATE inline
1227 unsigned
1228 Thread::migration_helper(Migration_info const *inf)
1229 {
1230   // LOG_MSG_3VAL(this, "MGi ", Mword(current()), (current_cpu() << 16) | cpu(), 0);
1231   assert_kdb (cpu() == current_cpu());
1232   assert_kdb (current() != this);
1233   assert_kdb (cpu_lock.test());
1234
1235   if (_timeout)
1236     _timeout->reset();
1237   ready_dequeue();
1238
1239     {
1240       // Not sure if this can ever happen
1241       Sched_context *csc = Context::current_sched();
1242       if (!csc || csc->context() == this)
1243         Context::set_current_sched(current()->sched());
1244     }
1245
1246   unsigned cpu = inf->cpu;
1247
1248     {
1249       Queue &q = _pending_rqq.cpu(current_cpu());
1250       // The queue lock of the current CPU protects the cpu number in
1251       // the thread
1252       Lock_guard<Pending_rqq::Inner_lock> g(q.q_lock());
1253
1254       // potentailly dequeue from our local queue
1255       if (_pending_rq.queued())
1256         check_kdb (q.dequeue(&_pending_rq, Queue_item::Ok));
1257
1258       Sched_context *sc = sched_context();
1259       sc->set_prio(inf->prio);
1260       sc->set_quantum(inf->quantum);
1261       sc->replenish();
1262       set_sched(sc);
1263
1264       if (drq_pending())
1265         state_add_dirty(Thread_drq_ready);
1266
1267       Mem::mp_wmb();
1268
1269       assert_kdb (!in_ready_list());
1270
1271       set_cpu_of(this, cpu);
1272       // now we are migrated away fom current_cpu
1273     }
1274
1275   bool ipi = true;
1276
1277     {
1278       Queue &q = _pending_rqq.cpu(cpu);
1279       Lock_guard<Pending_rqq::Inner_lock> g(q.q_lock());
1280
1281       // migrated meanwhile
1282       if (this->cpu() != cpu || _pending_rq.queued())
1283         return  Drq::No_answer | Drq::Need_resched;
1284
1285       if (q.first())
1286         ipi = false;
1287
1288       q.enqueue(&_pending_rq);
1289     }
1290
1291   if (ipi)
1292     {
1293       //LOG_MSG_3VAL(this, "sipi", current_cpu(), cpu(), (Mword)current());
1294       Ipi::cpu(cpu).send(Ipi::Request);
1295     }
1296
1297   return  Drq::No_answer | Drq::Need_resched;
1298 }
1299
1300 PRIVATE inline
1301 void
1302 Thread::migrate_xcpu(unsigned cpu)
1303 {
1304   bool ipi = true;
1305
1306     {
1307       Queue &q = Context::_pending_rqq.cpu(cpu);
1308       Lock_guard<Pending_rqq::Inner_lock> g(q.q_lock());
1309
1310       // already migrated
1311       if (cpu != this->cpu())
1312         return;
1313
1314       if (q.first())
1315         ipi = false;
1316
1317       if (!_pending_rq.queued())
1318         q.enqueue(&_pending_rq);
1319       else
1320         ipi = false;
1321     }
1322
1323   if (ipi)
1324     Ipi::cpu(cpu).send(Ipi::Request);
1325 }
1326
1327 //----------------------------------------------------------------------------
1328 IMPLEMENTATION [debug]:
1329
1330 IMPLEMENT
1331 unsigned
1332 Thread::Migration_log::fmt(Tb_entry *e, int maxlen, char *buf)
1333 {
1334   Migration_log *l = e->payload<Migration_log>();
1335   return snprintf(buf, maxlen, "migrate from %u to %u (state=%lx user ip=%lx)",
1336       l->src_cpu, l->target_cpu, l->state, l->user_ip);
1337 }
1338