kernel/fiasco/src/kern/thread-ipc.cpp

   1 INTERFACE [debug]:
   2
   3 #include "tb_entry.h"
   4
   5 EXTENSION class Thread
   6 {
   7 protected:
   8   struct Log_pf_invalid : public Tb_entry
   9   {
  10     Mword pfa;
  11     Mword cap_idx;
  12     Mword err;
  13     unsigned print(int max, char *buf) const;
  14   };
  15
  16   struct Log_exc_invalid : public Tb_entry
  17   {
  18     Mword cap_idx;
  19     unsigned print(int max, char *buf) const;
  20   };
  21 };
  22
  23 INTERFACE:
  24
  25 #include "l4_buf_iter.h"
  26 #include "l4_error.h"
  27
  28 class Syscall_frame;
  29
  30 EXTENSION class Thread
  31 {
  32 protected:
  33   enum Check_sender_result
  34   {
  35     Ok = 0,
  36     Queued = 2,
  37     Done = 4,
  38     Failed = 1,
  39   };
  40
  41   Syscall_frame *_snd_regs;
  42   unsigned char _ipc_send_rights;
  43 };
  44
  45 class Buf_utcb_saver
  46 {
  47 public:
  48   Buf_utcb_saver(Utcb const *u);
  49   void restore(Utcb *u);
  50 private:
  51   L4_buf_desc buf_desc;
  52   Mword buf[2];
  53 };
  54
  55 /**
  56  * Save critical contents of UTCB during nested IPC.
  57  */
  58 class Pf_msg_utcb_saver : public Buf_utcb_saver
  59 {
  60 public:
  61   Pf_msg_utcb_saver(Utcb const *u);
  62   void restore(Utcb *u);
  63 private:
  64   Mword msg[2];
  65 };
  66
  67 struct Ipc_remote_request;
  68
  69 struct Ipc_remote_request
  70 {
  71   L4_msg_tag tag;
  72   Thread *partner;
  73   Syscall_frame *regs;
  74   unsigned char rights;
  75   bool timeout;
  76   bool have_rcv;
  77
  78   unsigned result;
  79 };
  80
  81 struct Ready_queue_request
  82 {
  83   Thread *thread;
  84   Mword state_add;
  85   Mword state_del;
  86
  87   enum Result { Done, Wrong_cpu, Not_existent };
  88   Result result;
  89 };
  90
  91
  92 // ------------------------------------------------------------------------
  93 INTERFACE [debug]:
  94
  95 #include "tb_entry.h"
  96
  97 EXTENSION class Thread
  98 {
  99 protected:
 100   static unsigned log_fmt_pf_invalid(Tb_entry *, int max, char *buf) asm ("__fmt_page_fault_invalid_pager");
 101   static unsigned log_fmt_exc_invalid(Tb_entry *, int max, char *buf) asm ("__fmt_exception_invalid_handler");
 102 };
 103
 104 // ------------------------------------------------------------------------
 105 IMPLEMENTATION:
 106
 107 // IPC setup, and handling of ``short IPC'' and page-fault IPC
 108
 109 // IDEAS for enhancing this implementation:
 110
 111 // Volkmar has suggested a possible optimization for
 112 // short-flexpage-to-long-message-buffer transfers: Currently, we have
 113 // to resort to long IPC in that case because the message buffer might
 114 // contain a receive-flexpage option.  An easy optimization would be
 115 // to cache the receive-flexpage option in the TCB for that case.
 116 // This would save us the long-IPC setup because we wouldn't have to
 117 // touch the receiver's user memory in that case.  Volkmar argues that
 118 // cases like that are quite common -- for example, imagine a pager
 119 // which at the same time is also a server for ``normal'' requests.
 120
 121 // The handling of cancel and timeout conditions could be improved as
 122 // follows: Cancel and Timeout should not reset the ipc_in_progress
 123 // flag.  Instead, they should just set and/or reset a flag of their
 124 // own that is checked every time an (IPC) system call wants to go to
 125 // sleep.  That would mean that IPCs that do not block are not
 126 // cancelled or aborted.
 127 //-
 128
 129 #include <cstdlib>              // panic()
 130
 131 #include "l4_types.h"
 132 #include "l4_msg_item.h"
 133
 134 #include "config.h"
 135 #include "cpu_lock.h"
 136 #include "ipc_timeout.h"
 137 #include "lock_guard.h"
 138 #include "logdefs.h"
 139 #include "map_util.h"
 140 #include "processor.h"
 141 #include "timer.h"
 142 #include "kdb_ke.h"
 143 #include "warn.h"
 144
 145 PUBLIC
 146 virtual void
 147 Thread::ipc_receiver_aborted()
 148 {
 149   assert_kdb (wait_queue());
 150   set_wait_queue(0);
 151
 152   // remote_ready_enqueue(): is only for mp
 153   activate();
 154 }
 155
 156 PRIVATE
 157 void
 158 Thread::ipc_send_msg(Receiver *recv)
 159 {
 160   Syscall_frame *regs = _snd_regs;
 161   bool success = transfer_msg(regs->tag(), nonull_static_cast<Thread*>(recv), regs,
 162                               _ipc_send_rights);
 163   sender_dequeue(recv->sender_list());
 164   recv->vcpu_update_state();
 165   //printf("  done\n");
 166   regs->tag(L4_msg_tag(regs->tag(), success ? 0 : L4_msg_tag::Error));
 167
 168   Mword state_del = Thread_ipc_mask | Thread_ipc_transfer;
 169   Mword state_add = Thread_ready;
 170   if (Receiver::prepared())
 171     // same as in Receiver::prepare_receive_dirty_2
 172     state_add |= Thread_receive_wait;
 173
 174   if (cpu() == current_cpu())
 175     {
 176       state_change_dirty(~state_del, state_add);
 177       auto &rq = Sched_context::rq.current();
 178       Sched_context *cs = rq.current_sched();
 179       if (rq.deblock(cs, cs, true))
 180         recv->switch_to_locked(this);
 181     }
 182   else
 183     {
 184       drq_state_change(~state_del, state_add);
 185       current()->schedule_if(current()->handle_drq());
 186     }
 187 }
 188
 189 PUBLIC virtual
 190 void
 191 Thread::modify_label(Mword const *todo, int cnt)
 192 {
 193   assert_kdb (_snd_regs);
 194   Mword l = _snd_regs->from_spec();
 195   for (int i = 0; i < cnt*4; i += 4)
 196     {
 197       Mword const test_mask = todo[i];
 198       Mword const test      = todo[i+1];
 199       if ((l & test_mask) == test)
 200         {
 201           Mword const del_mask = todo[i+2];
 202           Mword const add_mask = todo[i+3];
 203
 204           l = (l & ~del_mask) | add_mask;
 205           _snd_regs->from(l);
 206           return;
 207         }
 208     }
 209 }
 210
 211 PRIVATE inline
 212 void
 213 Thread::snd_regs(Syscall_frame *r)
 214 { _snd_regs = r; }
 215
 216
 217 /** Page fault handler.
 218     This handler suspends any ongoing IPC, then sets up page-fault IPC.
 219     Finally, the ongoing IPC's state (if any) is restored.
 220     @param pfa page-fault virtual address
 221     @param error_code page-fault error code.
 222  */
 223 PRIVATE
 224 bool
 225 Thread::handle_page_fault_pager(Thread_ptr const &_pager,
 226                                 Address pfa, Mword error_code,
 227                                 L4_msg_tag::Protocol protocol)
 228 {
 229 #ifndef NDEBUG
 230   // do not handle user space page faults from kernel mode if we're
 231   // already handling a request
 232   if (EXPECT_FALSE(!PF::is_usermode_error(error_code)
 233                    && thread_lock()->test() == Thread_lock::Locked))
 234     {
 235       kdb_ke("Fiasco BUG: page fault, under lock");
 236       panic("page fault in locked operation");
 237     }
 238 #endif
 239
 240   if (EXPECT_FALSE((state() & Thread_alien)))
 241     return false;
 242
 243   auto guard = lock_guard(cpu_lock);
 244
 245   unsigned char rights;
 246   Kobject_iface *pager = _pager.ptr(space(), &rights);
 247
 248   if (!pager)
 249     {
 250       WARN("CPU%d: Pager of %lx is invalid (pfa=" L4_PTR_FMT
 251            ", errorcode=" L4_PTR_FMT ") to %lx (pc=%lx)\n",
 252            current_cpu(), dbg_id(), pfa, error_code,
 253            _pager.raw(), regs()->ip());
 254
 255
 256       LOG_TRACE("Page fault invalid pager", "pf", this, Log_pf_invalid,
 257                 l->cap_idx = _pager.raw();
 258                 l->err     = error_code;
 259                 l->pfa     = pfa);
 260
 261       pager = this; // block on ourselves
 262     }
 263
 264   // set up a register block used as an IPC parameter block for the
 265   // page fault IPC
 266   Syscall_frame r;
 267
 268   // save the UTCB fields affected by PF IPC
 269   Mword vcpu_irqs = vcpu_disable_irqs();
 270   Mem::barrier();
 271   Utcb *utcb = this->utcb().access(true);
 272   Pf_msg_utcb_saver saved_utcb_fields(utcb);
 273
 274
 275   utcb->buf_desc = L4_buf_desc(0, 0, 0, L4_buf_desc::Inherit_fpu);
 276   utcb->buffers[0] = L4_msg_item::map(0).raw();
 277   utcb->buffers[1] = L4_fpage::all_spaces().raw();
 278
 279   utcb->values[0] = PF::addr_to_msgword0 (pfa, error_code);
 280   utcb->values[1] = regs()->ip(); //PF::pc_to_msgword1 (regs()->ip(), error_code));
 281
 282   L4_timeout_pair timeout(L4_timeout::Never, L4_timeout::Never);
 283
 284   L4_msg_tag tag(2, 0, 0, protocol);
 285
 286   r.timeout(timeout);
 287   r.tag(tag);
 288   r.from(0);
 289   r.ref(L4_obj_ref(_pager.raw() << L4_obj_ref::Cap_shift, L4_obj_ref::Ipc_call_ipc));
 290   pager->invoke(r.ref(), rights, &r, utcb);
 291
 292
 293   bool success = true;
 294
 295   if (EXPECT_FALSE(r.tag().has_error()))
 296     {
 297       if (utcb->error.snd_phase()
 298           && (utcb->error.error() == L4_error::Not_existent)
 299           && PF::is_usermode_error(error_code)
 300           && !(state() & Thread_cancel))
 301         {
 302           success = false;
 303         }
 304     }
 305   else // no error
 306     {
 307       // If the pager rejects the mapping, it replies -1 in msg.w0
 308       if (EXPECT_FALSE (utcb->values[0] == Mword(-1)))
 309         success = false;
 310     }
 311
 312   // restore previous IPC state
 313
 314   saved_utcb_fields.restore(utcb);
 315   Mem::barrier();
 316   vcpu_restore_irqs(vcpu_irqs);
 317   return success;
 318 }
 319
 320 PRIVATE inline
 321 Mword
 322 Thread::check_sender(Thread *sender, bool timeout)
 323 {
 324   if (EXPECT_FALSE(is_invalid()))
 325     {
 326       sender->utcb().access()->error = L4_error::Not_existent;
 327       return Failed;
 328     }
 329
 330   if (EXPECT_FALSE(!sender_ok(sender)))
 331     {
 332       if (!timeout)
 333         {
 334           sender->utcb().access()->error = L4_error::Timeout;
 335           return Failed;
 336         }
 337
 338       sender->set_wait_queue(sender_list());
 339       sender->sender_enqueue(sender_list(), sender->sched_context()->prio());
 340       vcpu_set_irq_pending();
 341       return Queued;
 342     }
 343
 344   return Ok;
 345 }
 346
 347
 348 PRIVATE inline NEEDS["timer.h"]
 349 void Thread::goto_sleep(L4_timeout const &t, Sender *sender, Utcb *utcb)
 350 {
 351   IPC_timeout timeout;
 352
 353   if (EXPECT_FALSE(t.is_finite() && !_timeout))
 354     {
 355       state_del_dirty(Thread_ready);
 356
 357       Unsigned64 sysclock = Timer::system_clock();
 358       Unsigned64 tval = t.microsecs(sysclock, utcb);
 359
 360       if (EXPECT_TRUE((tval > sysclock)))
 361         {
 362           set_timeout(&timeout);
 363           timeout.set(tval, cpu(true));
 364         }
 365       else // timeout already hit
 366         state_change_dirty(~Thread_ipc_mask, Thread_ready | Thread_timeout);
 367     }
 368   else
 369     {
 370       if (EXPECT_TRUE(t.is_never()))
 371         state_del_dirty(Thread_ready);
 372       else
 373         state_change_dirty(~Thread_ipc_mask, Thread_ready | Thread_timeout);
 374     }
 375
 376   if (sender == this)
 377     switch_sched(sched(), &Sched_context::rq.current());
 378
 379   schedule();
 380
 381   if (EXPECT_FALSE((long)_timeout))
 382     {
 383       timeout.reset();
 384       set_timeout(0);
 385     }
 386
 387   assert_kdb (state() & Thread_ready);
 388 }
 389
 390
 391
 392 /**
 393  * @pre cpu_lock must be held
 394  */
 395 PRIVATE inline NEEDS["logdefs.h"]
 396 unsigned
 397 Thread::handshake_receiver(Thread *partner, L4_timeout snd_t)
 398 {
 399   assert_kdb(cpu_lock.test());
 400
 401   switch (__builtin_expect(partner->check_sender(this, !snd_t.is_zero()), Ok))
 402     {
 403     case Failed:
 404       return Failed;
 405     case Queued:
 406       state_add_dirty(Thread_send_wait);
 407       return Queued;
 408     default:
 409       partner->state_change_dirty(~(Thread_ipc_mask | Thread_ready), Thread_ipc_transfer);
 410       return Ok;
 411     }
 412 }
 413
 414 PRIVATE inline
 415 void
 416 Thread::set_ipc_error(L4_error const &e, Thread *rcv)
 417 {
 418   utcb().access()->error = e;
 419   rcv->utcb().access()->error = L4_error(e, L4_error::Rcv);
 420 }
 421
 422
 423 PRIVATE inline
 424 Sender *
 425 Thread::get_next_sender(Sender *sender)
 426 {
 427   if (!sender_list()->empty())
 428     {
 429       if (sender) // closed wait
 430         {
 431           if (sender->in_sender_list() && sender_list() == sender->wait_queue())
 432             return sender;
 433         }
 434       else // open wait
 435         {
 436           Sender *next = Sender::cast(sender_list()->first());
 437           assert_kdb (next->in_sender_list());
 438           set_partner(next);
 439           return next;
 440         }
 441     }
 442   return 0;
 443 }
 444
 445
 446 /**
 447  * Send an IPC message.
 448  *        Block until we can send the message or the timeout hits.
 449  * @param partner the receiver of our message
 450  * @param t a timeout specifier
 451  * @param regs sender's IPC registers
 452  * @pre cpu_lock must be held
 453  * @return sender's IPC error code
 454  */
 455 PUBLIC
 456 void
 457 Thread::do_ipc(L4_msg_tag const &tag, bool have_send, Thread *partner,
 458                bool have_receive, Sender *sender,
 459                L4_timeout_pair t, Syscall_frame *regs,
 460                unsigned char rights)
 461 {
 462   assert_kdb (cpu_lock.test());
 463   assert_kdb (this == current());
 464
 465   bool do_switch = false;
 466
 467   assert_kdb (!(state() & Thread_ipc_mask));
 468
 469   prepare_receive(sender, have_receive ? regs : 0);
 470   bool activate_partner = false;
 471
 472   if (have_send)
 473     {
 474       assert_kdb(!in_sender_list());
 475       do_switch = tag.do_switch();
 476
 477       bool ok;
 478       unsigned result;
 479
 480       set_ipc_send_rights(rights);
 481
 482       if (EXPECT_FALSE(partner->cpu() != current_cpu()) ||
 483           ((result = handshake_receiver(partner, t.snd)) == Failed
 484            && partner->drq_pending()))
 485         {
 486           // we have either per se X-CPU IPC or we ran into a
 487           // IPC during migration (indicated by the pending DRQ)
 488           do_switch = false;
 489           result = remote_handshake_receiver(tag, partner, have_receive, t.snd,
 490                                              regs, rights);
 491         }
 492
 493       switch (__builtin_expect(result, Ok))
 494         {
 495         case Done:
 496           ok = true;
 497           break;
 498
 499         case Queued:
 500           // set _snd_regs, to enable active receiving
 501           snd_regs(regs);
 502           ok = do_send_wait(partner, t.snd); // --- blocking point ---
 503           break;
 504
 505         case Failed:
 506           state_del_dirty(Thread_ipc_mask);
 507           ok = false;
 508           break;
 509
 510         default:
 511           // mmh, we can reset the receivers timeout
 512           // ping pong with timeouts will profit from it, because
 513           // it will require much less sorting overhead
 514           // if we dont reset the timeout, the possibility is very high
 515           // that the receiver timeout is in the timeout queue
 516           partner->reset_timeout();
 517
 518           ok = transfer_msg(tag, partner, regs, rights);
 519
 520           // switch to receiving state
 521           state_del_dirty(Thread_ipc_mask);
 522           if (ok && have_receive)
 523             state_add_dirty(Thread_receive_wait);
 524
 525           activate_partner = partner != this;
 526           break;
 527         }
 528
 529       if (EXPECT_FALSE(!ok))
 530         {
 531           // send failed, so do not switch to receiver directly and skip receive phase
 532           have_receive = false;
 533           regs->tag(L4_msg_tag(0, 0, L4_msg_tag::Error, 0));
 534         }
 535     }
 536   else
 537     {
 538       assert_kdb (have_receive);
 539       state_add_dirty(Thread_receive_wait);
 540     }
 541
 542   // only do direct switch on closed wait (call) or if we run on a foreign
 543   // scheduling context
 544   Sender *next = 0;
 545
 546   have_receive = state() & Thread_receive_wait;
 547
 548   if (have_receive)
 549     {
 550       assert_kdb (!in_sender_list());
 551       assert_kdb (!(state() & Thread_send_wait));
 552       next = get_next_sender(sender);
 553     }
 554
 555   if (activate_partner)
 556     {
 557       if (partner->cpu() == current_cpu())
 558         {
 559           auto &rq = Sched_context::rq.current();
 560           Sched_context *cs = rq.current_sched();
 561           do_switch = do_switch && ((have_receive && sender) || cs->context() != this) && !next;
 562           partner->state_change_dirty(~Thread_ipc_transfer, Thread_ready);
 563           if (do_switch)
 564             {
 565               if (handle_drq())
 566                 {
 567                   rq.deblock(partner->sched(), cs, false);
 568                   schedule();
 569                 }
 570               else
 571                 schedule_if(switch_exec_locked(partner, Context::Not_Helping));
 572             }
 573           else if (rq.deblock(partner->sched(), cs, true))
 574             switch_to_locked(partner);
 575         }
 576       else
 577         partner->drq_state_change(~Thread_ipc_transfer, Thread_ready);
 578     }
 579
 580   if (next)
 581     {
 582       state_change_dirty(~Thread_ipc_mask, Thread_receive_in_progress);
 583       next->ipc_send_msg(this);
 584       state_del_dirty(Thread_ipc_mask);
 585     }
 586   else if (have_receive)
 587     {
 588       if ((state() & Thread_full_ipc_mask) == Thread_receive_wait)
 589         goto_sleep(t.rcv, sender, utcb().access(true));
 590     }
 591
 592   if (EXPECT_TRUE (!(state() & Thread_full_ipc_mask)))
 593     return;
 594
 595   while (EXPECT_FALSE(state() & Thread_ipc_transfer))
 596     {
 597       state_del_dirty(Thread_ready);
 598       schedule();
 599     }
 600
 601   if (EXPECT_TRUE (!(state() & Thread_full_ipc_mask)))
 602     return;
 603
 604   Utcb *utcb = this->utcb().access(true);
 605   // the IPC has not been finished.  could be timeout or cancel
 606   // XXX should only modify the error-code part of the status code
 607
 608   if (EXPECT_FALSE(state() & Thread_cancel))
 609     {
 610       // we've presumably been reset!
 611       regs->tag(commit_error(utcb, L4_error::R_canceled, regs->tag()));
 612     }
 613   else
 614     regs->tag(commit_error(utcb, L4_error::R_timeout, regs->tag()));
 615   state_del(Thread_full_ipc_mask);
 616 }
 617
 618
 619 PRIVATE inline NEEDS [Thread::copy_utcb_to]
 620 bool
 621 Thread::transfer_msg(L4_msg_tag tag, Thread *receiver,
 622                      Syscall_frame *sender_regs, unsigned char rights)
 623 {
 624   Syscall_frame* dst_regs = receiver->rcv_regs();
 625
 626   bool success = copy_utcb_to(tag, receiver, rights);
 627   tag.set_error(!success);
 628   dst_regs->tag(tag);
 629   dst_regs->from(sender_regs->from_spec());
 630
 631   // setup the reply capability in case of a call
 632   if (success && partner() == receiver)
 633     receiver->set_caller(this, rights);
 634
 635   return success;
 636 }
 637
 638
 639
 640 IMPLEMENT inline
 641 Buf_utcb_saver::Buf_utcb_saver(const Utcb *u)
 642 {
 643   buf_desc = u->buf_desc;
 644   buf[0] = u->buffers[0];
 645   buf[1] = u->buffers[1];
 646 }
 647
 648 IMPLEMENT inline
 649 void
 650 Buf_utcb_saver::restore(Utcb *u)
 651 {
 652   u->buf_desc = buf_desc;
 653   u->buffers[0] = buf[0];
 654   u->buffers[1] = buf[1];
 655 }
 656
 657 IMPLEMENT inline
 658 Pf_msg_utcb_saver::Pf_msg_utcb_saver(Utcb const *u) : Buf_utcb_saver(u)
 659 {
 660   msg[0] = u->values[0];
 661   msg[1] = u->values[1];
 662 }
 663
 664 IMPLEMENT inline
 665 void
 666 Pf_msg_utcb_saver::restore(Utcb *u)
 667 {
 668   Buf_utcb_saver::restore(u);
 669   u->values[0] = msg[0];
 670   u->values[1] = msg[1];
 671 }
 672
 673
 674 /**
 675  * \pre must run with local IRQs disabled (CPU lock held)
 676  * to ensure that handler does not dissapear meanwhile.
 677  */
 678 PRIVATE
 679 bool
 680 Thread::exception(Kobject_iface *handler, Trap_state *ts, Mword rights)
 681 {
 682   Syscall_frame r;
 683   L4_timeout_pair timeout(L4_timeout::Never, L4_timeout::Never);
 684
 685   CNT_EXC_IPC;
 686
 687   Mword vcpu_irqs = vcpu_disable_irqs();
 688   Mem::barrier();
 689
 690   void *old_utcb_handler = _utcb_handler;
 691   _utcb_handler = ts;
 692
 693   // fill registers for IPC
 694   Utcb *utcb = this->utcb().access(true);
 695   Buf_utcb_saver saved_state(utcb);
 696
 697   utcb->buf_desc = L4_buf_desc(0, 0, 0, L4_buf_desc::Inherit_fpu);
 698   utcb->buffers[0] = L4_msg_item::map(0).raw();
 699   utcb->buffers[1] = L4_fpage::all_spaces().raw();
 700
 701   // clear regs
 702   L4_msg_tag tag(L4_exception_ipc::Msg_size, 0, L4_msg_tag::Transfer_fpu,
 703                  L4_msg_tag::Label_exception);
 704
 705   r.tag(tag);
 706   r.timeout(timeout);
 707   r.from(0);
 708   r.ref(L4_obj_ref(_exc_handler.raw() << L4_obj_ref::Cap_shift, L4_obj_ref::Ipc_call_ipc));
 709   spill_user_state();
 710   handler->invoke(r.ref(), rights, &r, utcb);
 711   fill_user_state();
 712
 713   saved_state.restore(utcb);
 714
 715   if (EXPECT_FALSE(r.tag().has_error()))
 716     state_del(Thread_in_exception);
 717   else if (r.tag().proto() == L4_msg_tag::Label_allow_syscall)
 718     state_add(Thread_dis_alien);
 719
 720   // restore original utcb_handler
 721   _utcb_handler = old_utcb_handler;
 722   Mem::barrier();
 723   vcpu_restore_irqs(vcpu_irqs);
 724
 725   // FIXME: handle not existing pager properly
 726   // for now, just ignore any errors
 727   return 1;
 728 }
 729
 730 /* return 1 if exception could be handled
 731  * return 0 if not for send_exception and halt thread
 732  */
 733 PUBLIC inline NEEDS["task.h", "trap_state.h",
 734                     Thread::fast_return_to_user,
 735                     Thread::save_fpu_state_to_utcb]
 736 int
 737 Thread::send_exception(Trap_state *ts)
 738 {
 739   assert(cpu_lock.test());
 740
 741   Vcpu_state *vcpu = vcpu_state().access();
 742
 743   if (vcpu_exceptions_enabled(vcpu))
 744     {
 745       // do not reflect debug exceptions to the VCPU but handle them in
 746       // Fiasco
 747       if (EXPECT_FALSE(ts->is_debug_exception()
 748                        && !(vcpu->state & Vcpu_state::F_debug_exc)))
 749         return 0;
 750
 751       if (_exc_cont.valid())
 752         return 1;
 753
 754       // before entering kernel mode to have original fpu state before
 755       // enabling fpu
 756       save_fpu_state_to_utcb(ts, utcb().access());
 757
 758       spill_user_state();
 759
 760       if (vcpu_enter_kernel_mode(vcpu))
 761         {
 762           // enter_kernel_mode has switched the address space from user to
 763           // kernel space, so reevaluate the address of the VCPU state area
 764           vcpu = vcpu_state().access();
 765         }
 766
 767       LOG_TRACE("VCPU events", "vcpu", this, Vcpu_log,
 768           l->type = 2;
 769           l->state = vcpu->_saved_state;
 770           l->ip = ts->ip();
 771           l->sp = ts->sp();
 772           l->trap = ts->trapno();
 773           l->err = ts->error();
 774           l->space = vcpu_user_space() ? static_cast<Task*>(vcpu_user_space())->dbg_id() : ~0;
 775           );
 776       memcpy(&vcpu->_ts, ts, sizeof(Trap_state));
 777       fast_return_to_user(vcpu->_entry_ip, vcpu->_sp, vcpu_state().usr().get());
 778     }
 779
 780   // local IRQs must be disabled because we dereference a Thread_ptr
 781   if (EXPECT_FALSE(_exc_handler.is_kernel()))
 782     return 0;
 783
 784   if (!send_exception_arch(ts))
 785     return 0; // do not send exception
 786
 787   unsigned char rights = 0;
 788   Kobject_iface *pager = _exc_handler.ptr(space(), &rights);
 789
 790   if (EXPECT_FALSE(!pager))
 791     {
 792       /* no pager (anymore), just ignore the exception, return success */
 793       LOG_TRACE("Exception invalid handler", "exc", this, Log_exc_invalid,
 794                 l->cap_idx = _exc_handler.raw());
 795       if (EXPECT_FALSE(space()->is_sigma0()))
 796         {
 797           ts->dump();
 798           WARNX(Error, "Sigma0 raised an exception --> HALT\n");
 799           panic("...");
 800         }
 801
 802       pager = this; // block on ourselves
 803     }
 804
 805   state_change(~Thread_cancel, Thread_in_exception);
 806
 807   return exception(pager, ts, rights);
 808 }
 809
 810 PRIVATE static
 811 bool
 812 Thread::try_transfer_local_id(L4_buf_iter::Item const *const buf,
 813                               L4_fpage sfp, Mword *rcv_word, Thread* snd,
 814                               Thread *rcv)
 815 {
 816   if (buf->b.is_rcv_id())
 817     {
 818       if (snd->space() == rcv->space())
 819         {
 820           rcv_word[-2] |= 6;
 821           rcv_word[-1] = sfp.raw();
 822           return true;
 823         }
 824       else
 825         {
 826           unsigned char rights = 0;
 827           Obj_space::Capability cap = snd->space()->lookup(sfp.obj_index());
 828           Kobject_iface *o = cap.obj();
 829           rights = cap.rights();
 830           if (EXPECT_TRUE(o && o->is_local(rcv->space())))
 831             {
 832               rcv_word[-2] |= 4;
 833               rcv_word[-1] = o->obj_id() | Mword(rights);
 834               return true;
 835             }
 836         }
 837     }
 838   return false;
 839 }
 840
 841 PRIVATE static inline
 842 bool FIASCO_WARN_RESULT
 843 Thread::copy_utcb_to_utcb(L4_msg_tag const &tag, Thread *snd, Thread *rcv,
 844                           unsigned char rights)
 845 {
 846   assert (cpu_lock.test());
 847
 848   Utcb *snd_utcb = snd->utcb().access();
 849   Utcb *rcv_utcb = rcv->utcb().access();
 850   Mword s = tag.words();
 851   Mword r = Utcb::Max_words;
 852
 853   Mem::memcpy_mwords(rcv_utcb->values, snd_utcb->values, r < s ? r : s);
 854
 855   bool success = true;
 856   if (tag.items())
 857     success = transfer_msg_items(tag, snd, snd_utcb, rcv, rcv_utcb, rights);
 858
 859   if (tag.transfer_fpu() && rcv_utcb->inherit_fpu() && (rights & L4_fpage::W))
 860     snd->transfer_fpu(rcv);
 861
 862   return success;
 863 }
 864
 865
 866 PUBLIC inline NEEDS[Thread::copy_utcb_to_ts, Thread::copy_utcb_to_utcb,
 867                     Thread::copy_ts_to_utcb]
 868 bool FIASCO_WARN_RESULT
 869 Thread::copy_utcb_to(L4_msg_tag const &tag, Thread* receiver,
 870                      unsigned char rights)
 871 {
 872   // we cannot copy trap state to trap state!
 873   assert_kdb (!this->_utcb_handler || !receiver->_utcb_handler);
 874   if (EXPECT_FALSE(this->_utcb_handler != 0))
 875     return copy_ts_to_utcb(tag, this, receiver, rights);
 876   else if (EXPECT_FALSE(receiver->_utcb_handler != 0))
 877     return copy_utcb_to_ts(tag, this, receiver, rights);
 878   else
 879     return copy_utcb_to_utcb(tag, this, receiver, rights);
 880 }
 881
 882 PRIVATE static
 883 bool
 884 Thread::transfer_msg_items(L4_msg_tag const &tag, Thread* snd, Utcb *snd_utcb,
 885                            Thread *rcv, Utcb *rcv_utcb,
 886                            unsigned char rights)
 887 {
 888   // LOG_MSG_3VAL(current(), "map bd=", rcv_utcb->buf_desc.raw(), 0, 0);
 889   Task *const rcv_t = nonull_static_cast<Task*>(rcv->space());
 890   L4_buf_iter mem_buffer(rcv_utcb, rcv_utcb->buf_desc.mem());
 891   L4_buf_iter io_buffer(rcv_utcb, rcv_utcb->buf_desc.io());
 892   L4_buf_iter obj_buffer(rcv_utcb, rcv_utcb->buf_desc.obj());
 893   L4_snd_item_iter snd_item(snd_utcb, tag.words());
 894   register int items = tag.items();
 895   Mword *rcv_word = rcv_utcb->values + tag.words();
 896
 897   // XXX: damn X-CPU state modification
 898   // snd->prepare_long_ipc(rcv);
 899   Reap_list rl;
 900
 901   for (;items > 0 && snd_item.more();)
 902     {
 903       if (EXPECT_FALSE(!snd_item.next()))
 904         {
 905           snd->set_ipc_error(L4_error::Overflow, rcv);
 906           return false;
 907         }
 908
 909       L4_snd_item_iter::Item const *const item = snd_item.get();
 910
 911       if (item->b.is_void())
 912         { // XXX: not sure if void fpages are needed
 913           // skip send item and current rcv_buffer
 914           --items;
 915           continue;
 916         }
 917
 918       L4_buf_iter *buf_iter = 0;
 919
 920       switch (item->b.type())
 921         {
 922         case L4_msg_item::Map:
 923           switch (L4_fpage(item->d).type())
 924             {
 925             case L4_fpage::Memory: buf_iter = &mem_buffer; break;
 926             case L4_fpage::Io:     buf_iter = &io_buffer; break;
 927             case L4_fpage::Obj:    buf_iter = &obj_buffer; break;
 928             default: break;
 929             }
 930           break;
 931         default:
 932           break;
 933         }
 934
 935       if (EXPECT_FALSE(!buf_iter))
 936         {
 937           // LOG_MSG_3VAL(snd, "lIPCm0", 0, 0, 0);
 938           snd->set_ipc_error(L4_error::Overflow, rcv);
 939           return false;
 940         }
 941
 942       L4_buf_iter::Item const *const buf = buf_iter->get();
 943
 944       if (EXPECT_FALSE(buf->b.is_void() || buf->b.type() != item->b.type()))
 945         {
 946           // LOG_MSG_3VAL(snd, "lIPCm1", buf->b.raw(), item->b.raw(), 0);
 947           snd->set_ipc_error(L4_error::Overflow, rcv);
 948           return false;
 949         }
 950
 951         {
 952           assert_kdb (item->b.type() == L4_msg_item::Map);
 953           L4_fpage sfp(item->d);
 954           *rcv_word = (item->b.raw() & ~0x0ff7) | (sfp.raw() & 0x0ff0);
 955
 956           rcv_word += 2;
 957
 958           if (!try_transfer_local_id(buf, sfp, rcv_word, snd, rcv))
 959             {
 960               // we need to do a real mapping¿
 961
 962               // diminish when sending via restricted ipc gates
 963               if (sfp.type() == L4_fpage::Obj)
 964                 sfp.mask_rights(L4_fpage::Rights(rights | L4_fpage::RX));
 965
 966               L4_error err;
 967
 968                 {
 969                   // We take the existence_lock for syncronizing maps...
 970                   // This is kind of coarse grained
 971                   Lock_guard<decltype(rcv_t->existence_lock)> sp_lock;
 972                   if (!sp_lock.check_and_lock(&rcv_t->existence_lock))
 973                     {
 974                       snd->set_ipc_error(L4_error::Overflow, rcv);
 975                       return false;
 976                     }
 977
 978                   auto c_lock = lock_guard(cpu_lock);
 979                   err = fpage_map(snd->space(), sfp,
 980                       rcv->space(), L4_fpage(buf->d), item->b, &rl);
 981                 }
 982
 983               if (EXPECT_FALSE(!err.ok()))
 984                 {
 985                   snd->set_ipc_error(err, rcv);
 986                   return false;
 987                 }
 988             }
 989         }
 990
 991       --items;
 992
 993       if (!item->b.compound())
 994         buf_iter->next();
 995     }
 996
 997   if (EXPECT_FALSE(items))
 998     {
 999       snd->set_ipc_error(L4_error::Overflow, rcv);
1000       return false;
1001     }
1002
1003   return true;
1004 }
1005
1006
1007 /**
1008  * \pre Runs on the sender CPU
1009  */
1010 PRIVATE inline
1011 bool
1012 Thread::abort_send(L4_error const &e, Thread *partner)
1013 {
1014   state_del_dirty(Thread_full_ipc_mask);
1015
1016   if (_timeout && _timeout->is_set())
1017     _timeout->reset();
1018
1019   set_timeout(0);
1020   Abort_state abt = Abt_ipc_done;
1021
1022   if (partner->cpu() == current_cpu())
1023     {
1024       if (in_sender_list())
1025         {
1026           sender_dequeue(partner->sender_list());
1027           partner->vcpu_update_state();
1028           abt = Abt_ipc_cancel;
1029
1030         }
1031       else if (partner->in_ipc(this))
1032         abt = Abt_ipc_in_progress;
1033     }
1034   else
1035     abt = partner->Receiver::abort_send(this);
1036
1037   switch (abt)
1038     {
1039     default:
1040     case Abt_ipc_done:
1041       return true;
1042     case Abt_ipc_cancel:
1043       utcb().access()->error = e;
1044       return false;
1045     case Abt_ipc_in_progress:
1046       state_add_dirty(Thread_ipc_transfer);
1047       while (state() & Thread_ipc_transfer)
1048         {
1049           state_del_dirty(Thread_ready);
1050           schedule();
1051         }
1052       return true;
1053     }
1054 }
1055
1056
1057
1058 /**
1059  * \pre Runs on the sender CPU
1060  */
1061 PRIVATE inline
1062 bool
1063 Thread::do_send_wait(Thread *partner, L4_timeout snd_t)
1064 {
1065   IPC_timeout timeout;
1066
1067   if (EXPECT_FALSE(snd_t.is_finite()))
1068     {
1069       Unsigned64 tval = snd_t.microsecs(Timer::system_clock(), utcb().access(true));
1070       // Zero timeout or timeout expired already -- give up
1071       if (tval == 0)
1072         return abort_send(L4_error::Timeout, partner);
1073
1074       set_timeout(&timeout);
1075       timeout.set(tval, cpu());
1076     }
1077
1078   register Mword ipc_state;
1079
1080   while (((ipc_state = state() & (Thread_send_wait | Thread_ipc_abort_mask))) == Thread_send_wait)
1081     {
1082       state_del_dirty(Thread_ready);
1083       schedule();
1084     }
1085
1086   if (EXPECT_FALSE(ipc_state == (Thread_cancel | Thread_send_wait)))
1087     return abort_send(L4_error::Canceled, partner);
1088
1089   if (EXPECT_FALSE(ipc_state == (Thread_timeout | Thread_send_wait)))
1090     return abort_send(L4_error::Timeout, partner);
1091
1092   timeout.reset();
1093   set_timeout(0);
1094
1095   return true;
1096 }
1097
1098 PRIVATE inline
1099 void
1100 Thread::set_ipc_send_rights(unsigned char c)
1101 {
1102   _ipc_send_rights = c;
1103 }
1104
1105 PRIVATE inline NOEXPORT
1106 bool
1107 Thread::remote_ipc_send(Context *src, Ipc_remote_request *rq)
1108 {
1109   (void)src;
1110   // LOG_MSG_3VAL(this, "rse", current_cpu(), (Mword)src, (Mword)this);
1111 #if 0
1112   LOG_MSG_3VAL(this, "rsend", (Mword)src, 0, 0);
1113   printf("CPU[%u]: remote IPC send ...\n"
1114          "  partner=%p [%u]\n"
1115          "  sender =%p [%u] regs=%p\n"
1116          "  timeout=%u\n",
1117          current_cpu(),
1118          rq->partner, rq->partner->cpu(),
1119          src, src->cpu(),
1120          rq->regs,
1121          rq->timeout);
1122 #endif
1123
1124   switch (__builtin_expect(rq->partner->check_sender(this, rq->timeout), Ok))
1125     {
1126     case Failed:
1127       rq->result = Failed;
1128       return false;
1129     case Queued:
1130       rq->result = Queued;
1131       return false;
1132     default:
1133       break;
1134     }
1135
1136   // trigger remote_ipc_receiver_ready path, because we may need to grab locks
1137   // and this is forbidden in a DRQ handler. So transfer the IPC in usual
1138   // thread code. However, this induces a overhead of two extra IPIs.
1139   if (rq->tag.items())
1140     {
1141       //LOG_MSG_3VAL(rq->partner, "pull", dbg_id(), 0, 0);
1142       rq->partner->state_change_dirty(~(Thread_ipc_mask | Thread_ready), Thread_ipc_transfer);
1143       rq->result = Ok;
1144       return true;
1145     }
1146   bool success = transfer_msg(rq->tag, rq->partner, rq->regs, _ipc_send_rights);
1147   rq->result = success ? Done : Failed;
1148
1149   rq->partner->state_change_dirty(~Thread_ipc_mask, Thread_ready);
1150   // hm, should be done by lazy queueing: rq->partner->ready_enqueue();
1151   return true;
1152 }
1153
1154 PRIVATE static
1155 unsigned
1156 Thread::handle_remote_ipc_send(Drq *src, Context *, void *_rq)
1157 {
1158   Ipc_remote_request *rq = (Ipc_remote_request*)_rq;
1159   bool r = nonull_static_cast<Thread*>(src->context())->remote_ipc_send(src->context(), rq);
1160   //LOG_MSG_3VAL(src, "rse<", current_cpu(), (Mword)src, r);
1161   return r ? Drq::Need_resched : 0;
1162 }
1163
1164 /**
1165  * \pre Runs on the sender CPU
1166  */
1167 PRIVATE //inline NEEDS ["mp_request.h"]
1168 unsigned
1169 Thread::remote_handshake_receiver(L4_msg_tag const &tag, Thread *partner,
1170                                   bool have_receive,
1171                                   L4_timeout snd_t, Syscall_frame *regs,
1172                                   unsigned char rights)
1173 {
1174   // Flag that there must be no switch in the receive path.
1175   // This flag also prevents the receive path from accessing
1176   // the thread state of a remote sender.
1177   Ipc_remote_request rq;
1178   rq.tag = tag;
1179   rq.have_rcv = have_receive;
1180   rq.partner = partner;
1181   rq.timeout = !snd_t.is_zero();
1182   rq.regs = regs;
1183   rq.rights = rights;
1184   _snd_regs = regs;
1185
1186   set_wait_queue(partner->sender_list());
1187
1188   state_add_dirty(Thread_send_wait);
1189
1190   partner->drq(handle_remote_ipc_send, &rq,
1191                remote_prepare_receive);
1192
1193   return rq.result;
1194 }
1195
1196 PRIVATE static
1197 unsigned
1198 Thread::remote_prepare_receive(Drq *src, Context *, void *arg)
1199 {
1200   Context *c = src->context();
1201   Ipc_remote_request *rq = (Ipc_remote_request*)arg;
1202   //printf("CPU[%2u:%p]: remote_prepare_receive (err=%x)\n", current_cpu(), c, rq->err.error());
1203
1204   // No atomic switch to receive state if we are queued, or the IPC must be done by
1205   // the sender's CPU
1206   if (EXPECT_FALSE(rq->result == Queued || rq->result == Ok))
1207     return 0;
1208
1209   c->state_del(Thread_ipc_mask);
1210   if (EXPECT_FALSE((rq->result & Failed) || !rq->have_rcv))
1211     return 0;
1212
1213   c->state_add_dirty(Thread_receive_wait);
1214   return 0;
1215 }
1216
1217 //---------------------------------------------------------------------------
1218 IMPLEMENTATION [debug]:
1219
1220 IMPLEMENT
1221 unsigned
1222 Thread::Log_pf_invalid::print(int max, char *buf) const
1223 {
1224   return snprintf(buf, max, "InvCap C:%lx pfa=%lx err=%lx", cap_idx, pfa, err);
1225 }
1226
1227 IMPLEMENT
1228 unsigned
1229 Thread::Log_exc_invalid::print(int max, char *buf) const
1230 {
1231   return snprintf(buf, max, "InvCap C:%lx", cap_idx);
1232 }