kernel/fiasco/src/kern/thread-ipc.cpp

   1 INTERFACE [debug]:
   2
   3 #include "tb_entry.h"
   4
   5 EXTENSION class Thread
   6 {
   7 protected:
   8   struct Log_pf_invalid : public Tb_entry
   9   {
  10     Mword pfa;
  11     Cap_index cap_idx;
  12     Mword err;
  13     void print(String_buffer *buf) const;
  14   };
  15
  16   struct Log_exc_invalid : public Tb_entry
  17   {
  18     Cap_index cap_idx;
  19     void print(String_buffer *buf) const;
  20   };
  21 };
  22
  23 INTERFACE:
  24
  25 #include "l4_buf_iter.h"
  26 #include "l4_error.h"
  27
  28 class Syscall_frame;
  29
  30 EXTENSION class Thread
  31 {
  32 protected:
  33   enum Check_sender_result
  34   {
  35     Ok = 0,
  36     Queued = 2,
  37     Done = 4,
  38     Failed = 1,
  39   };
  40
  41   Syscall_frame *_snd_regs;
  42   L4_fpage::Rights _ipc_send_rights;
  43 };
  44
  45 class Buf_utcb_saver
  46 {
  47 public:
  48   Buf_utcb_saver(Utcb const *u);
  49   void restore(Utcb *u);
  50 private:
  51   L4_buf_desc buf_desc;
  52   Mword buf[2];
  53 };
  54
  55 /**
  56  * Save critical contents of UTCB during nested IPC.
  57  */
  58 class Pf_msg_utcb_saver : public Buf_utcb_saver
  59 {
  60 public:
  61   Pf_msg_utcb_saver(Utcb const *u);
  62   void restore(Utcb *u);
  63 private:
  64   Mword msg[2];
  65 };
  66
  67 struct Ipc_remote_request;
  68
  69 struct Ipc_remote_request
  70 {
  71   L4_msg_tag tag;
  72   Thread *partner;
  73   Syscall_frame *regs;
  74   L4_fpage::Rights rights;
  75   bool timeout;
  76   bool have_rcv;
  77
  78   unsigned result;
  79 };
  80
  81 struct Ready_queue_request
  82 {
  83   Thread *thread;
  84   Mword state_add;
  85   Mword state_del;
  86
  87   enum Result { Done, Wrong_cpu, Not_existent };
  88   Result result;
  89 };
  90
  91
  92 // ------------------------------------------------------------------------
  93 INTERFACE [debug]:
  94
  95 #include "tb_entry.h"
  96
  97 EXTENSION class Thread
  98 {
  99 protected:
 100   static unsigned log_fmt_pf_invalid(Tb_entry *, int max, char *buf) asm ("__fmt_page_fault_invalid_pager");
 101   static unsigned log_fmt_exc_invalid(Tb_entry *, int max, char *buf) asm ("__fmt_exception_invalid_handler");
 102 };
 103
 104 // ------------------------------------------------------------------------
 105 IMPLEMENTATION:
 106
 107 // IPC setup, and handling of ``short IPC'' and page-fault IPC
 108
 109 // IDEAS for enhancing this implementation:
 110
 111 // Volkmar has suggested a possible optimization for
 112 // short-flexpage-to-long-message-buffer transfers: Currently, we have
 113 // to resort to long IPC in that case because the message buffer might
 114 // contain a receive-flexpage option.  An easy optimization would be
 115 // to cache the receive-flexpage option in the TCB for that case.
 116 // This would save us the long-IPC setup because we wouldn't have to
 117 // touch the receiver's user memory in that case.  Volkmar argues that
 118 // cases like that are quite common -- for example, imagine a pager
 119 // which at the same time is also a server for ``normal'' requests.
 120
 121 // The handling of cancel and timeout conditions could be improved as
 122 // follows: Cancel and Timeout should not reset the ipc_in_progress
 123 // flag.  Instead, they should just set and/or reset a flag of their
 124 // own that is checked every time an (IPC) system call wants to go to
 125 // sleep.  That would mean that IPCs that do not block are not
 126 // cancelled or aborted.
 127 //-
 128
 129 #include <cstdlib>              // panic()
 130
 131 #include "l4_types.h"
 132 #include "l4_msg_item.h"
 133
 134 #include "config.h"
 135 #include "cpu_lock.h"
 136 #include "ipc_timeout.h"
 137 #include "lock_guard.h"
 138 #include "logdefs.h"
 139 #include "map_util.h"
 140 #include "processor.h"
 141 #include "timer.h"
 142 #include "kdb_ke.h"
 143 #include "warn.h"
 144
 145 PUBLIC
 146 virtual void
 147 Thread::ipc_receiver_aborted()
 148 {
 149   assert_kdb (wait_queue());
 150   set_wait_queue(0);
 151
 152   // remote_ready_enqueue(): is only for mp
 153   activate();
 154 }
 155
 156 PRIVATE
 157 void
 158 Thread::ipc_send_msg(Receiver *recv)
 159 {
 160   Syscall_frame *regs = _snd_regs;
 161   bool success = transfer_msg(regs->tag(), nonull_static_cast<Thread*>(recv), regs,
 162                               _ipc_send_rights);
 163   sender_dequeue(recv->sender_list());
 164   recv->vcpu_update_state();
 165   //printf("  done\n");
 166   regs->tag(L4_msg_tag(regs->tag(), success ? 0 : L4_msg_tag::Error));
 167
 168   Mword state_del = Thread_ipc_mask | Thread_ipc_transfer;
 169   Mword state_add = Thread_ready;
 170   if (Receiver::prepared())
 171     // same as in Receiver::prepare_receive_dirty_2
 172     state_add |= Thread_receive_wait;
 173
 174   if (home_cpu() == current_cpu())
 175     {
 176       state_change_dirty(~state_del, state_add);
 177       auto &rq = Sched_context::rq.current();
 178       Sched_context *cs = rq.current_sched();
 179       if (rq.deblock(cs, cs, true))
 180         recv->switch_to_locked(this);
 181     }
 182   else
 183     {
 184       drq_state_change(~state_del, state_add);
 185       current()->schedule_if(current()->handle_drq());
 186     }
 187 }
 188
 189 PUBLIC virtual
 190 void
 191 Thread::modify_label(Mword const *todo, int cnt)
 192 {
 193   assert_kdb (_snd_regs);
 194   Mword l = _snd_regs->from_spec();
 195   for (int i = 0; i < cnt*4; i += 4)
 196     {
 197       Mword const test_mask = todo[i];
 198       Mword const test      = todo[i+1];
 199       if ((l & test_mask) == test)
 200         {
 201           Mword const del_mask = todo[i+2];
 202           Mword const add_mask = todo[i+3];
 203
 204           l = (l & ~del_mask) | add_mask;
 205           _snd_regs->from(l);
 206           return;
 207         }
 208     }
 209 }
 210
 211 PRIVATE inline
 212 void
 213 Thread::snd_regs(Syscall_frame *r)
 214 { _snd_regs = r; }
 215
 216
 217 /** Page fault handler.
 218     This handler suspends any ongoing IPC, then sets up page-fault IPC.
 219     Finally, the ongoing IPC's state (if any) is restored.
 220     @param pfa page-fault virtual address
 221     @param error_code page-fault error code.
 222  */
 223 PRIVATE
 224 bool
 225 Thread::handle_page_fault_pager(Thread_ptr const &_pager,
 226                                 Address pfa, Mword error_code,
 227                                 L4_msg_tag::Protocol protocol)
 228 {
 229   if (EXPECT_FALSE((state() & Thread_alien)))
 230     return false;
 231
 232   auto guard = lock_guard(cpu_lock);
 233
 234   L4_fpage::Rights rights;
 235   Kobject_iface *pager = _pager.ptr(space(), &rights);
 236
 237   if (!pager)
 238     {
 239       WARN("CPU%d: Pager of %lx is invalid (pfa=" L4_PTR_FMT
 240            ", errorcode=" L4_PTR_FMT ") to %lx (pc=%lx)\n",
 241            cxx::int_value<Cpu_number>(current_cpu()), dbg_id(), pfa,
 242            error_code, cxx::int_value<Cap_index>(_pager.raw()), regs()->ip());
 243
 244
 245       LOG_TRACE("Page fault invalid pager", "pf", this, Log_pf_invalid,
 246                 l->cap_idx = _pager.raw();
 247                 l->err     = error_code;
 248                 l->pfa     = pfa);
 249
 250       pager = this; // block on ourselves
 251     }
 252
 253   // set up a register block used as an IPC parameter block for the
 254   // page fault IPC
 255   Syscall_frame r;
 256
 257   // save the UTCB fields affected by PF IPC
 258   Mword vcpu_irqs = vcpu_disable_irqs();
 259   Mem::barrier();
 260   Utcb *utcb = this->utcb().access(true);
 261   Pf_msg_utcb_saver saved_utcb_fields(utcb);
 262
 263
 264   utcb->buf_desc = L4_buf_desc(0, 0, 0, L4_buf_desc::Inherit_fpu);
 265   utcb->buffers[0] = L4_msg_item::map(0).raw();
 266   utcb->buffers[1] = L4_fpage::all_spaces().raw();
 267
 268   utcb->values[0] = PF::addr_to_msgword0 (pfa, error_code);
 269   utcb->values[1] = regs()->ip(); //PF::pc_to_msgword1 (regs()->ip(), error_code));
 270
 271   L4_timeout_pair timeout(L4_timeout::Never, L4_timeout::Never);
 272
 273   L4_msg_tag tag(2, 0, 0, protocol);
 274
 275   r.timeout(timeout);
 276   r.tag(tag);
 277   r.from(0);
 278   r.ref(L4_obj_ref(_pager.raw(), L4_obj_ref::Ipc_call_ipc));
 279   pager->invoke(r.ref(), rights, &r, utcb);
 280
 281
 282   bool success = true;
 283
 284   if (EXPECT_FALSE(r.tag().has_error()))
 285     {
 286       if (utcb->error.snd_phase()
 287           && (utcb->error.error() == L4_error::Not_existent)
 288           && PF::is_usermode_error(error_code)
 289           && !(state() & Thread_cancel))
 290         {
 291           success = false;
 292         }
 293     }
 294   else // no error
 295     {
 296       // If the pager rejects the mapping, it replies -1 in msg.w0
 297       if (EXPECT_FALSE (utcb->values[0] == Mword(-1)))
 298         success = false;
 299     }
 300
 301   // restore previous IPC state
 302
 303   saved_utcb_fields.restore(utcb);
 304   Mem::barrier();
 305   vcpu_restore_irqs(vcpu_irqs);
 306   return success;
 307 }
 308
 309 PRIVATE inline
 310 Mword
 311 Thread::check_sender(Thread *sender, bool timeout)
 312 {
 313   if (EXPECT_FALSE(is_invalid()))
 314     {
 315       sender->utcb().access()->error = L4_error::Not_existent;
 316       return Failed;
 317     }
 318
 319   if (EXPECT_FALSE(!sender_ok(sender)))
 320     {
 321       if (!timeout)
 322         {
 323           sender->utcb().access()->error = L4_error::Timeout;
 324           return Failed;
 325         }
 326
 327       sender->set_wait_queue(sender_list());
 328       sender->sender_enqueue(sender_list(), sender->sched_context()->prio());
 329       vcpu_set_irq_pending();
 330       return Queued;
 331     }
 332
 333   return Ok;
 334 }
 335
 336
 337 PRIVATE inline NEEDS["timer.h"]
 338 void Thread::goto_sleep(L4_timeout const &t, Sender *sender, Utcb *utcb)
 339 {
 340   IPC_timeout timeout;
 341
 342   if (EXPECT_FALSE(t.is_finite() && !_timeout))
 343     {
 344       state_del_dirty(Thread_ready);
 345
 346       Unsigned64 sysclock = Timer::system_clock();
 347       Unsigned64 tval = t.microsecs(sysclock, utcb);
 348
 349       if (EXPECT_TRUE((tval > sysclock)))
 350         {
 351           set_timeout(&timeout);
 352           timeout.set(tval, current_cpu());
 353         }
 354       else // timeout already hit
 355         state_change_dirty(~Thread_ipc_mask, Thread_ready | Thread_timeout);
 356     }
 357   else
 358     {
 359       if (EXPECT_TRUE(t.is_never()))
 360         state_del_dirty(Thread_ready);
 361       else
 362         state_change_dirty(~Thread_ipc_mask, Thread_ready | Thread_timeout);
 363     }
 364
 365   if (sender == this)
 366     switch_sched(sched(), &Sched_context::rq.current());
 367
 368   schedule();
 369
 370   if (EXPECT_FALSE((long)_timeout))
 371     {
 372       timeout.reset();
 373       set_timeout(0);
 374     }
 375
 376   assert_kdb (state() & Thread_ready);
 377 }
 378
 379
 380
 381 /**
 382  * @pre cpu_lock must be held
 383  */
 384 PRIVATE inline NEEDS["logdefs.h"]
 385 unsigned
 386 Thread::handshake_receiver(Thread *partner, L4_timeout snd_t)
 387 {
 388   assert_kdb(cpu_lock.test());
 389
 390   switch (__builtin_expect(partner->check_sender(this, !snd_t.is_zero()), Ok))
 391     {
 392     case Failed:
 393       return Failed;
 394     case Queued:
 395       state_add_dirty(Thread_send_wait);
 396       return Queued;
 397     default:
 398       partner->state_change_dirty(~(Thread_ipc_mask | Thread_ready), Thread_ipc_transfer);
 399       return Ok;
 400     }
 401 }
 402
 403 PRIVATE inline
 404 void
 405 Thread::set_ipc_error(L4_error const &e, Thread *rcv)
 406 {
 407   utcb().access()->error = e;
 408   rcv->utcb().access()->error = L4_error(e, L4_error::Rcv);
 409 }
 410
 411
 412 PRIVATE inline
 413 Sender *
 414 Thread::get_next_sender(Sender *sender)
 415 {
 416   if (!sender_list()->empty())
 417     {
 418       if (sender) // closed wait
 419         {
 420           if (sender->in_sender_list() && sender_list() == sender->wait_queue())
 421             return sender;
 422         }
 423       else // open wait
 424         {
 425           Sender *next = Sender::cast(sender_list()->first());
 426           assert_kdb (next->in_sender_list());
 427           set_partner(next);
 428           return next;
 429         }
 430     }
 431   return 0;
 432 }
 433
 434
 435 /**
 436  * Send an IPC message.
 437  *        Block until we can send the message or the timeout hits.
 438  * @param partner the receiver of our message
 439  * @param t a timeout specifier
 440  * @param regs sender's IPC registers
 441  * @pre cpu_lock must be held
 442  * @return sender's IPC error code
 443  */
 444 PUBLIC
 445 void
 446 Thread::do_ipc(L4_msg_tag const &tag, bool have_send, Thread *partner,
 447                bool have_receive, Sender *sender,
 448                L4_timeout_pair t, Syscall_frame *regs,
 449                L4_fpage::Rights rights)
 450 {
 451   assert_kdb (cpu_lock.test());
 452   assert_kdb (this == current());
 453
 454   bool do_switch = false;
 455
 456   assert_kdb (!(state() & Thread_ipc_mask));
 457
 458   prepare_receive(sender, have_receive ? regs : 0);
 459   bool activate_partner = false;
 460
 461   if (have_send)
 462     {
 463       assert_kdb(!in_sender_list());
 464       do_switch = tag.do_switch();
 465
 466       bool ok;
 467       unsigned result;
 468
 469       set_ipc_send_rights(rights);
 470
 471       if (EXPECT_FALSE(partner->home_cpu() != current_cpu()) ||
 472           ((result = handshake_receiver(partner, t.snd)) == Failed
 473            && partner->drq_pending()))
 474         {
 475           // we have either per se X-CPU IPC or we ran into a
 476           // IPC during migration (indicated by the pending DRQ)
 477           do_switch = false;
 478           result = remote_handshake_receiver(tag, partner, have_receive, t.snd,
 479                                              regs, rights);
 480         }
 481
 482       switch (__builtin_expect(result, Ok))
 483         {
 484         case Done:
 485           ok = true;
 486           break;
 487
 488         case Queued:
 489           // set _snd_regs, to enable active receiving
 490           snd_regs(regs);
 491           ok = do_send_wait(partner, t.snd); // --- blocking point ---
 492           break;
 493
 494         case Failed:
 495           state_del_dirty(Thread_ipc_mask);
 496           ok = false;
 497           break;
 498
 499         default:
 500           // mmh, we can reset the receivers timeout
 501           // ping pong with timeouts will profit from it, because
 502           // it will require much less sorting overhead
 503           // if we dont reset the timeout, the possibility is very high
 504           // that the receiver timeout is in the timeout queue
 505           partner->reset_timeout();
 506
 507           ok = transfer_msg(tag, partner, regs, rights);
 508
 509           // switch to receiving state
 510           state_del_dirty(Thread_ipc_mask);
 511           if (ok && have_receive)
 512             state_add_dirty(Thread_receive_wait);
 513
 514           activate_partner = partner != this;
 515           break;
 516         }
 517
 518       if (EXPECT_FALSE(!ok))
 519         {
 520           // send failed, so do not switch to receiver directly and skip receive phase
 521           have_receive = false;
 522           regs->tag(L4_msg_tag(0, 0, L4_msg_tag::Error, 0));
 523         }
 524     }
 525   else
 526     {
 527       assert_kdb (have_receive);
 528       state_add_dirty(Thread_receive_wait);
 529     }
 530
 531   // only do direct switch on closed wait (call) or if we run on a foreign
 532   // scheduling context
 533   Sender *next = 0;
 534
 535   have_receive = state() & Thread_receive_wait;
 536
 537   if (have_receive)
 538     {
 539       assert_kdb (!in_sender_list());
 540       assert_kdb (!(state() & Thread_send_wait));
 541       next = get_next_sender(sender);
 542     }
 543
 544   if (activate_partner)
 545     {
 546       if (partner->home_cpu() == current_cpu())
 547         {
 548           auto &rq = Sched_context::rq.current();
 549           Sched_context *cs = rq.current_sched();
 550           do_switch = do_switch && ((have_receive && sender) || cs->context() != this) && !next;
 551           partner->state_change_dirty(~Thread_ipc_transfer, Thread_ready);
 552           if (do_switch)
 553             {
 554               if (handle_drq())
 555                 {
 556                   rq.deblock(partner->sched(), cs, false);
 557                   schedule();
 558                 }
 559               else
 560                 schedule_if(switch_exec_locked(partner, Context::Not_Helping));
 561             }
 562           else if (rq.deblock(partner->sched(), cs, true))
 563             switch_to_locked(partner);
 564         }
 565       else
 566         partner->drq_state_change(~Thread_ipc_transfer, Thread_ready);
 567     }
 568
 569   if (next)
 570     {
 571       state_change_dirty(~Thread_ipc_mask, Thread_receive_in_progress);
 572       next->ipc_send_msg(this);
 573       state_del_dirty(Thread_ipc_mask);
 574     }
 575   else if (have_receive)
 576     {
 577       if ((state() & Thread_full_ipc_mask) == Thread_receive_wait)
 578         goto_sleep(t.rcv, sender, utcb().access(true));
 579     }
 580
 581   if (EXPECT_TRUE (!(state() & Thread_full_ipc_mask)))
 582     return;
 583
 584   while (EXPECT_FALSE(state() & Thread_ipc_transfer))
 585     {
 586       state_del_dirty(Thread_ready);
 587       schedule();
 588     }
 589
 590   if (EXPECT_TRUE (!(state() & Thread_full_ipc_mask)))
 591     return;
 592
 593   Utcb *utcb = this->utcb().access(true);
 594   // the IPC has not been finished.  could be timeout or cancel
 595   // XXX should only modify the error-code part of the status code
 596
 597   if (EXPECT_FALSE(state() & Thread_cancel))
 598     {
 599       // we've presumably been reset!
 600       regs->tag(commit_error(utcb, L4_error::R_canceled, regs->tag()));
 601     }
 602   else
 603     regs->tag(commit_error(utcb, L4_error::R_timeout, regs->tag()));
 604   state_del(Thread_full_ipc_mask);
 605 }
 606
 607
 608 PRIVATE inline NEEDS [Thread::copy_utcb_to]
 609 bool
 610 Thread::transfer_msg(L4_msg_tag tag, Thread *receiver,
 611                      Syscall_frame *sender_regs, L4_fpage::Rights rights)
 612 {
 613   Syscall_frame* dst_regs = receiver->rcv_regs();
 614
 615   bool success = copy_utcb_to(tag, receiver, rights);
 616   tag.set_error(!success);
 617   dst_regs->tag(tag);
 618   dst_regs->from(sender_regs->from_spec());
 619
 620   // setup the reply capability in case of a call
 621   if (success && partner() == receiver)
 622     receiver->set_caller(this, rights);
 623
 624   return success;
 625 }
 626
 627
 628
 629 IMPLEMENT inline
 630 Buf_utcb_saver::Buf_utcb_saver(const Utcb *u)
 631 {
 632   buf_desc = u->buf_desc;
 633   buf[0] = u->buffers[0];
 634   buf[1] = u->buffers[1];
 635 }
 636
 637 IMPLEMENT inline
 638 void
 639 Buf_utcb_saver::restore(Utcb *u)
 640 {
 641   u->buf_desc = buf_desc;
 642   u->buffers[0] = buf[0];
 643   u->buffers[1] = buf[1];
 644 }
 645
 646 IMPLEMENT inline
 647 Pf_msg_utcb_saver::Pf_msg_utcb_saver(Utcb const *u) : Buf_utcb_saver(u)
 648 {
 649   msg[0] = u->values[0];
 650   msg[1] = u->values[1];
 651 }
 652
 653 IMPLEMENT inline
 654 void
 655 Pf_msg_utcb_saver::restore(Utcb *u)
 656 {
 657   Buf_utcb_saver::restore(u);
 658   u->values[0] = msg[0];
 659   u->values[1] = msg[1];
 660 }
 661
 662
 663 /**
 664  * \pre must run with local IRQs disabled (CPU lock held)
 665  * to ensure that handler does not dissapear meanwhile.
 666  */
 667 PRIVATE
 668 bool
 669 Thread::exception(Kobject_iface *handler, Trap_state *ts, L4_fpage::Rights rights)
 670 {
 671   Syscall_frame r;
 672   L4_timeout_pair timeout(L4_timeout::Never, L4_timeout::Never);
 673
 674   CNT_EXC_IPC;
 675
 676   Mword vcpu_irqs = vcpu_disable_irqs();
 677   Mem::barrier();
 678
 679   void *old_utcb_handler = _utcb_handler;
 680   _utcb_handler = ts;
 681
 682   // fill registers for IPC
 683   Utcb *utcb = this->utcb().access(true);
 684   Buf_utcb_saver saved_state(utcb);
 685
 686   utcb->buf_desc = L4_buf_desc(0, 0, 0, L4_buf_desc::Inherit_fpu);
 687   utcb->buffers[0] = L4_msg_item::map(0).raw();
 688   utcb->buffers[1] = L4_fpage::all_spaces().raw();
 689
 690   // clear regs
 691   L4_msg_tag tag(L4_exception_ipc::Msg_size, 0, L4_msg_tag::Transfer_fpu,
 692                  L4_msg_tag::Label_exception);
 693
 694   r.tag(tag);
 695   r.timeout(timeout);
 696   r.from(0);
 697   r.ref(L4_obj_ref(_exc_handler.raw(), L4_obj_ref::Ipc_call_ipc));
 698   spill_user_state();
 699   handler->invoke(r.ref(), rights, &r, utcb);
 700   fill_user_state();
 701
 702   saved_state.restore(utcb);
 703
 704   if (EXPECT_FALSE(r.tag().has_error()))
 705     state_del(Thread_in_exception);
 706   else if (r.tag().proto() == L4_msg_tag::Label_allow_syscall)
 707     state_add(Thread_dis_alien);
 708
 709   // restore original utcb_handler
 710   _utcb_handler = old_utcb_handler;
 711   Mem::barrier();
 712   vcpu_restore_irqs(vcpu_irqs);
 713
 714   // FIXME: handle not existing pager properly
 715   // for now, just ignore any errors
 716   return 1;
 717 }
 718
 719 /* return 1 if exception could be handled
 720  * return 0 if not for send_exception and halt thread
 721  */
 722 PUBLIC inline NEEDS["task.h", "trap_state.h",
 723                     Thread::fast_return_to_user,
 724                     Thread::save_fpu_state_to_utcb]
 725 int
 726 Thread::send_exception(Trap_state *ts)
 727 {
 728   assert(cpu_lock.test());
 729
 730   Vcpu_state *vcpu = vcpu_state().access();
 731
 732   if (vcpu_exceptions_enabled(vcpu))
 733     {
 734       // do not reflect debug exceptions to the VCPU but handle them in
 735       // Fiasco
 736       if (EXPECT_FALSE(ts->is_debug_exception()
 737                        && !(vcpu->state & Vcpu_state::F_debug_exc)))
 738         return 0;
 739
 740       if (_exc_cont.valid())
 741         return 1;
 742
 743       // before entering kernel mode to have original fpu state before
 744       // enabling fpu
 745       save_fpu_state_to_utcb(ts, utcb().access());
 746
 747       spill_user_state();
 748
 749       if (vcpu_enter_kernel_mode(vcpu))
 750         {
 751           // enter_kernel_mode has switched the address space from user to
 752           // kernel space, so reevaluate the address of the VCPU state area
 753           vcpu = vcpu_state().access();
 754         }
 755
 756       LOG_TRACE("VCPU events", "vcpu", this, Vcpu_log,
 757           l->type = 2;
 758           l->state = vcpu->_saved_state;
 759           l->ip = ts->ip();
 760           l->sp = ts->sp();
 761           l->trap = ts->trapno();
 762           l->err = ts->error();
 763           l->space = vcpu_user_space() ? static_cast<Task*>(vcpu_user_space())->dbg_id() : ~0;
 764           );
 765       memcpy(&vcpu->_ts, ts, sizeof(Trap_state));
 766       fast_return_to_user(vcpu->_entry_ip, vcpu->_sp, vcpu_state().usr().get());
 767     }
 768
 769   // local IRQs must be disabled because we dereference a Thread_ptr
 770   if (EXPECT_FALSE(_exc_handler.is_kernel()))
 771     return 0;
 772
 773   if (!send_exception_arch(ts))
 774     return 0; // do not send exception
 775
 776   L4_fpage::Rights rights = L4_fpage::Rights(0);
 777   Kobject_iface *pager = _exc_handler.ptr(space(), &rights);
 778
 779   if (EXPECT_FALSE(!pager))
 780     {
 781       /* no pager (anymore), just ignore the exception, return success */
 782       LOG_TRACE("Exception invalid handler", "exc", this, Log_exc_invalid,
 783                 l->cap_idx = _exc_handler.raw());
 784       if (EXPECT_FALSE(space()->is_sigma0()))
 785         {
 786           ts->dump();
 787           WARNX(Error, "Sigma0 raised an exception --> HALT\n");
 788           panic("...");
 789         }
 790
 791       pager = this; // block on ourselves
 792     }
 793
 794   state_change(~Thread_cancel, Thread_in_exception);
 795
 796   return exception(pager, ts, rights);
 797 }
 798
 799 PRIVATE static
 800 bool
 801 Thread::try_transfer_local_id(L4_buf_iter::Item const *const buf,
 802                               L4_fpage sfp, Mword *rcv_word, Thread* snd,
 803                               Thread *rcv)
 804 {
 805   if (buf->b.is_rcv_id())
 806     {
 807       if (snd->space() == rcv->space())
 808         {
 809           rcv_word[-2] |= 6;
 810           rcv_word[-1] = sfp.raw();
 811           return true;
 812         }
 813       else
 814         {
 815           unsigned char rights = 0;
 816           Obj_space::Capability cap = snd->space()->lookup(sfp.obj_index());
 817           Kobject_iface *o = cap.obj();
 818           rights = cap.rights();
 819           if (EXPECT_TRUE(o && o->is_local(rcv->space())))
 820             {
 821               rcv_word[-2] |= 4;
 822               rcv_word[-1] = o->obj_id() | Mword(rights);
 823               return true;
 824             }
 825         }
 826     }
 827   return false;
 828 }
 829
 830 PRIVATE static inline
 831 bool FIASCO_WARN_RESULT
 832 Thread::copy_utcb_to_utcb(L4_msg_tag const &tag, Thread *snd, Thread *rcv,
 833                           L4_fpage::Rights rights)
 834 {
 835   assert (cpu_lock.test());
 836
 837   Utcb *snd_utcb = snd->utcb().access();
 838   Utcb *rcv_utcb = rcv->utcb().access();
 839   Mword s = tag.words();
 840   Mword r = Utcb::Max_words;
 841
 842   Mem::memcpy_mwords(rcv_utcb->values, snd_utcb->values, r < s ? r : s);
 843
 844   bool success = true;
 845   if (tag.items())
 846     success = transfer_msg_items(tag, snd, snd_utcb, rcv, rcv_utcb, rights);
 847
 848   if (tag.transfer_fpu() && rcv_utcb->inherit_fpu() && (rights & L4_fpage::Rights::W()))
 849     snd->transfer_fpu(rcv);
 850
 851   return success;
 852 }
 853
 854
 855 PUBLIC inline NEEDS[Thread::copy_utcb_to_ts, Thread::copy_utcb_to_utcb,
 856                     Thread::copy_ts_to_utcb]
 857 bool FIASCO_WARN_RESULT
 858 Thread::copy_utcb_to(L4_msg_tag const &tag, Thread* receiver,
 859                      L4_fpage::Rights rights)
 860 {
 861   // we cannot copy trap state to trap state!
 862   assert_kdb (!this->_utcb_handler || !receiver->_utcb_handler);
 863   if (EXPECT_FALSE(this->_utcb_handler != 0))
 864     return copy_ts_to_utcb(tag, this, receiver, rights);
 865   else if (EXPECT_FALSE(receiver->_utcb_handler != 0))
 866     return copy_utcb_to_ts(tag, this, receiver, rights);
 867   else
 868     return copy_utcb_to_utcb(tag, this, receiver, rights);
 869 }
 870
 871 PRIVATE static
 872 bool
 873 Thread::transfer_msg_items(L4_msg_tag const &tag, Thread* snd, Utcb *snd_utcb,
 874                            Thread *rcv, Utcb *rcv_utcb,
 875                            L4_fpage::Rights rights)
 876 {
 877   // LOG_MSG_3VAL(current(), "map bd=", rcv_utcb->buf_desc.raw(), 0, 0);
 878   Task *const rcv_t = nonull_static_cast<Task*>(rcv->space());
 879   L4_buf_iter mem_buffer(rcv_utcb, rcv_utcb->buf_desc.mem());
 880   L4_buf_iter io_buffer(rcv_utcb, rcv_utcb->buf_desc.io());
 881   L4_buf_iter obj_buffer(rcv_utcb, rcv_utcb->buf_desc.obj());
 882   L4_snd_item_iter snd_item(snd_utcb, tag.words());
 883   register int items = tag.items();
 884   Mword *rcv_word = rcv_utcb->values + tag.words();
 885
 886   // XXX: damn X-CPU state modification
 887   // snd->prepare_long_ipc(rcv);
 888   Reap_list rl;
 889
 890   for (;items > 0 && snd_item.more();)
 891     {
 892       if (EXPECT_FALSE(!snd_item.next()))
 893         {
 894           snd->set_ipc_error(L4_error::Overflow, rcv);
 895           return false;
 896         }
 897
 898       L4_snd_item_iter::Item const *const item = snd_item.get();
 899
 900       if (item->b.is_void())
 901         { // XXX: not sure if void fpages are needed
 902           // skip send item and current rcv_buffer
 903           --items;
 904           continue;
 905         }
 906
 907       L4_buf_iter *buf_iter = 0;
 908
 909       switch (item->b.type())
 910         {
 911         case L4_msg_item::Map:
 912           switch (L4_fpage(item->d).type())
 913             {
 914             case L4_fpage::Memory: buf_iter = &mem_buffer; break;
 915             case L4_fpage::Io:     buf_iter = &io_buffer; break;
 916             case L4_fpage::Obj:    buf_iter = &obj_buffer; break;
 917             default: break;
 918             }
 919           break;
 920         default:
 921           break;
 922         }
 923
 924       if (EXPECT_FALSE(!buf_iter))
 925         {
 926           // LOG_MSG_3VAL(snd, "lIPCm0", 0, 0, 0);
 927           snd->set_ipc_error(L4_error::Overflow, rcv);
 928           return false;
 929         }
 930
 931       L4_buf_iter::Item const *const buf = buf_iter->get();
 932
 933       if (EXPECT_FALSE(buf->b.is_void() || buf->b.type() != item->b.type()))
 934         {
 935           // LOG_MSG_3VAL(snd, "lIPCm1", buf->b.raw(), item->b.raw(), 0);
 936           snd->set_ipc_error(L4_error::Overflow, rcv);
 937           return false;
 938         }
 939
 940         {
 941           assert_kdb (item->b.type() == L4_msg_item::Map);
 942           L4_fpage sfp(item->d);
 943           *rcv_word = (item->b.raw() & ~0x0ff7) | (sfp.raw() & 0x0ff0);
 944
 945           rcv_word += 2;
 946
 947           if (!try_transfer_local_id(buf, sfp, rcv_word, snd, rcv))
 948             {
 949               // we need to do a real mapping¿
 950
 951               // diminish when sending via restricted ipc gates
 952               if (sfp.type() == L4_fpage::Obj)
 953                 sfp.mask_rights(rights | L4_fpage::Rights::CRW() | L4_fpage::Rights::CD());
 954
 955               L4_error err;
 956
 957                 {
 958                   // We take the existence_lock for syncronizing maps...
 959                   // This is kind of coarse grained
 960                   Lock_guard<decltype(rcv_t->existence_lock)> sp_lock;
 961                   if (!sp_lock.check_and_lock(&rcv_t->existence_lock))
 962                     {
 963                       snd->set_ipc_error(L4_error::Overflow, rcv);
 964                       return false;
 965                     }
 966
 967                   auto c_lock = lock_guard<Lock_guard_inverse_policy>(cpu_lock);
 968                   err = fpage_map(snd->space(), sfp,
 969                       rcv->space(), L4_fpage(buf->d), item->b, &rl);
 970                 }
 971
 972               if (EXPECT_FALSE(!err.ok()))
 973                 {
 974                   snd->set_ipc_error(err, rcv);
 975                   return false;
 976                 }
 977             }
 978         }
 979
 980       --items;
 981
 982       if (!item->b.compound())
 983         buf_iter->next();
 984     }
 985
 986   if (EXPECT_FALSE(items))
 987     {
 988       snd->set_ipc_error(L4_error::Overflow, rcv);
 989       return false;
 990     }
 991
 992   return true;
 993 }
 994
 995
 996 /**
 997  * \pre Runs on the sender CPU
 998  */
 999 PRIVATE inline
1000 bool
1001 Thread::abort_send(L4_error const &e, Thread *partner)
1002 {
1003   state_del_dirty(Thread_full_ipc_mask);
1004
1005   if (_timeout && _timeout->is_set())
1006     _timeout->reset();
1007
1008   set_timeout(0);
1009   Abort_state abt = Abt_ipc_done;
1010
1011   if (partner->home_cpu() == current_cpu())
1012     {
1013       if (in_sender_list())
1014         {
1015           sender_dequeue(partner->sender_list());
1016           partner->vcpu_update_state();
1017           abt = Abt_ipc_cancel;
1018
1019         }
1020       else if (partner->in_ipc(this))
1021         abt = Abt_ipc_in_progress;
1022     }
1023   else
1024     abt = partner->Receiver::abort_send(this);
1025
1026   switch (abt)
1027     {
1028     default:
1029     case Abt_ipc_done:
1030       return true;
1031     case Abt_ipc_cancel:
1032       utcb().access()->error = e;
1033       return false;
1034     case Abt_ipc_in_progress:
1035       state_add_dirty(Thread_ipc_transfer);
1036       while (state() & Thread_ipc_transfer)
1037         {
1038           state_del_dirty(Thread_ready);
1039           schedule();
1040         }
1041       return true;
1042     }
1043 }
1044
1045
1046
1047 /**
1048  * \pre Runs on the sender CPU
1049  */
1050 PRIVATE inline
1051 bool
1052 Thread::do_send_wait(Thread *partner, L4_timeout snd_t)
1053 {
1054   IPC_timeout timeout;
1055
1056   if (EXPECT_FALSE(snd_t.is_finite()))
1057     {
1058       Unsigned64 tval = snd_t.microsecs(Timer::system_clock(), utcb().access(true));
1059       // Zero timeout or timeout expired already -- give up
1060       if (tval == 0)
1061         return abort_send(L4_error::Timeout, partner);
1062
1063       set_timeout(&timeout);
1064       timeout.set(tval, current_cpu());
1065     }
1066
1067   register Mword ipc_state;
1068
1069   while (((ipc_state = state() & (Thread_send_wait | Thread_ipc_abort_mask))) == Thread_send_wait)
1070     {
1071       state_del_dirty(Thread_ready);
1072       schedule();
1073     }
1074
1075   if (EXPECT_FALSE(ipc_state == (Thread_cancel | Thread_send_wait)))
1076     return abort_send(L4_error::Canceled, partner);
1077
1078   if (EXPECT_FALSE(ipc_state == (Thread_timeout | Thread_send_wait)))
1079     return abort_send(L4_error::Timeout, partner);
1080
1081   timeout.reset();
1082   set_timeout(0);
1083
1084   return true;
1085 }
1086
1087 PRIVATE inline
1088 void
1089 Thread::set_ipc_send_rights(L4_fpage::Rights c)
1090 {
1091   _ipc_send_rights = c;
1092 }
1093
1094 PRIVATE inline NOEXPORT
1095 bool
1096 Thread::remote_ipc_send(Context *src, Ipc_remote_request *rq)
1097 {
1098   (void)src;
1099   // LOG_MSG_3VAL(this, "rse", current_cpu(), (Mword)src, (Mword)this);
1100 #if 0
1101   LOG_MSG_3VAL(this, "rsend", (Mword)src, 0, 0);
1102   printf("CPU[%u]: remote IPC send ...\n"
1103          "  partner=%p [%u]\n"
1104          "  sender =%p [%u] regs=%p\n"
1105          "  timeout=%u\n",
1106          current_cpu(),
1107          rq->partner, rq->partner->cpu(),
1108          src, src->cpu(),
1109          rq->regs,
1110          rq->timeout);
1111 #endif
1112
1113   switch (__builtin_expect(rq->partner->check_sender(this, rq->timeout), Ok))
1114     {
1115     case Failed:
1116       rq->result = Failed;
1117       return false;
1118     case Queued:
1119       rq->result = Queued;
1120       return false;
1121     default:
1122       break;
1123     }
1124
1125   // trigger remote_ipc_receiver_ready path, because we may need to grab locks
1126   // and this is forbidden in a DRQ handler. So transfer the IPC in usual
1127   // thread code. However, this induces a overhead of two extra IPIs.
1128   if (rq->tag.items())
1129     {
1130       //LOG_MSG_3VAL(rq->partner, "pull", dbg_id(), 0, 0);
1131       rq->partner->state_change_dirty(~(Thread_ipc_mask | Thread_ready), Thread_ipc_transfer);
1132       rq->result = Ok;
1133       return true;
1134     }
1135   bool success = transfer_msg(rq->tag, rq->partner, rq->regs, _ipc_send_rights);
1136   rq->result = success ? Done : Failed;
1137
1138   rq->partner->state_change_dirty(~Thread_ipc_mask, Thread_ready);
1139   // hm, should be done by lazy queueing: rq->partner->ready_enqueue();
1140   return true;
1141 }
1142
1143 PRIVATE static
1144 Context::Drq::Result
1145 Thread::handle_remote_ipc_send(Drq *src, Context *, void *_rq)
1146 {
1147   Ipc_remote_request *rq = (Ipc_remote_request*)_rq;
1148   bool r = nonull_static_cast<Thread*>(src->context())->remote_ipc_send(src->context(), rq);
1149   //LOG_MSG_3VAL(src, "rse<", current_cpu(), (Mword)src, r);
1150   return r ? Drq::need_resched() : Drq::done();
1151 }
1152
1153 /**
1154  * \pre Runs on the sender CPU
1155  */
1156 PRIVATE //inline NEEDS ["mp_request.h"]
1157 unsigned
1158 Thread::remote_handshake_receiver(L4_msg_tag const &tag, Thread *partner,
1159                                   bool have_receive,
1160                                   L4_timeout snd_t, Syscall_frame *regs,
1161                                   L4_fpage::Rights rights)
1162 {
1163   // Flag that there must be no switch in the receive path.
1164   // This flag also prevents the receive path from accessing
1165   // the thread state of a remote sender.
1166   Ipc_remote_request rq;
1167   rq.tag = tag;
1168   rq.have_rcv = have_receive;
1169   rq.partner = partner;
1170   rq.timeout = !snd_t.is_zero();
1171   rq.regs = regs;
1172   rq.rights = rights;
1173   _snd_regs = regs;
1174
1175   set_wait_queue(partner->sender_list());
1176
1177   state_add_dirty(Thread_send_wait);
1178
1179   partner->drq(handle_remote_ipc_send, &rq,
1180                remote_prepare_receive);
1181
1182   return rq.result;
1183 }
1184
1185 PRIVATE static
1186 Context::Drq::Result
1187 Thread::remote_prepare_receive(Drq *src, Context *, void *arg)
1188 {
1189   Context *c = src->context();
1190   Ipc_remote_request *rq = (Ipc_remote_request*)arg;
1191   //printf("CPU[%2u:%p]: remote_prepare_receive (err=%x)\n", current_cpu(), c, rq->err.error());
1192
1193   // No atomic switch to receive state if we are queued, or the IPC must be done by
1194   // the sender's CPU
1195   if (EXPECT_FALSE(rq->result == Queued || rq->result == Ok))
1196     return Drq::done();
1197
1198   c->state_del(Thread_ipc_mask);
1199   if (EXPECT_FALSE((rq->result & Failed) || !rq->have_rcv))
1200     return Drq::done();
1201
1202   c->state_add_dirty(Thread_receive_wait);
1203   return Drq::done();
1204 }
1205
1206 //---------------------------------------------------------------------------
1207 IMPLEMENTATION [debug]:
1208
1209 #include "string_buffer.h"
1210
1211 IMPLEMENT
1212 void
1213 Thread::Log_pf_invalid::print(String_buffer *buf) const
1214 {
1215   buf->printf("InvCap C:%lx pfa=%lx err=%lx",
1216               cxx::int_value<Cap_index>(cap_idx), pfa, err);
1217 }
1218
1219 IMPLEMENT
1220 void
1221 Thread::Log_exc_invalid::print(String_buffer *buf) const
1222 {
1223   buf->printf("InvCap C:%lx", cxx::int_value<Cap_index>(cap_idx));
1224 }