kernel/fiasco/src/kern/thread-ipc.cpp

   1 INTERFACE [debug]:
   2
   3 #include "tb_entry.h"
   4
   5 EXTENSION class Thread
   6 {
   7 protected:
   8   struct Log_pf_invalid : public Tb_entry
   9   {
  10     Mword pfa;
  11     Cap_index cap_idx;
  12     Mword err;
  13     void print(String_buffer *buf) const;
  14   };
  15
  16   struct Log_exc_invalid : public Tb_entry
  17   {
  18     Cap_index cap_idx;
  19     void print(String_buffer *buf) const;
  20   };
  21 };
  22
  23 INTERFACE:
  24
  25 #include "l4_buf_iter.h"
  26 #include "l4_error.h"
  27
  28 class Syscall_frame;
  29
  30 EXTENSION class Thread
  31 {
  32 protected:
  33   enum class Check_sender
  34   {
  35     Ok = 0,
  36     Queued = 2,
  37     Done = 4,
  38     Failed = 1,
  39   };
  40
  41   struct Ipc_remote_request
  42   {
  43     L4_msg_tag tag;
  44     Thread *partner;
  45     Syscall_frame *regs;
  46     L4_fpage::Rights rights;
  47     bool timeout;
  48     bool have_rcv;
  49
  50     Thread::Check_sender result;
  51   };
  52
  53   Syscall_frame *_snd_regs;
  54   L4_fpage::Rights _ipc_send_rights;
  55 };
  56
  57 class Buf_utcb_saver
  58 {
  59 public:
  60   Buf_utcb_saver(Utcb const *u);
  61   void restore(Utcb *u);
  62 private:
  63   L4_buf_desc buf_desc;
  64   Mword buf[2];
  65 };
  66
  67 /**
  68  * Save critical contents of UTCB during nested IPC.
  69  */
  70 class Pf_msg_utcb_saver : public Buf_utcb_saver
  71 {
  72 public:
  73   Pf_msg_utcb_saver(Utcb const *u);
  74   void restore(Utcb *u);
  75 private:
  76   Mword msg[2];
  77 };
  78
  79
  80 // ------------------------------------------------------------------------
  81 INTERFACE [debug]:
  82
  83 #include "tb_entry.h"
  84
  85 EXTENSION class Thread
  86 {
  87 protected:
  88   static unsigned log_fmt_pf_invalid(Tb_entry *, int max, char *buf) asm ("__fmt_page_fault_invalid_pager");
  89   static unsigned log_fmt_exc_invalid(Tb_entry *, int max, char *buf) asm ("__fmt_exception_invalid_handler");
  90 };
  91
  92 // ------------------------------------------------------------------------
  93 IMPLEMENTATION:
  94
  95 // IPC setup, and handling of ``short IPC'' and page-fault IPC
  96
  97 // IDEAS for enhancing this implementation:
  98
  99 // Volkmar has suggested a possible optimization for
 100 // short-flexpage-to-long-message-buffer transfers: Currently, we have
 101 // to resort to long IPC in that case because the message buffer might
 102 // contain a receive-flexpage option.  An easy optimization would be
 103 // to cache the receive-flexpage option in the TCB for that case.
 104 // This would save us the long-IPC setup because we wouldn't have to
 105 // touch the receiver's user memory in that case.  Volkmar argues that
 106 // cases like that are quite common -- for example, imagine a pager
 107 // which at the same time is also a server for ``normal'' requests.
 108
 109 // The handling of cancel and timeout conditions could be improved as
 110 // follows: Cancel and Timeout should not reset the ipc_in_progress
 111 // flag.  Instead, they should just set and/or reset a flag of their
 112 // own that is checked every time an (IPC) system call wants to go to
 113 // sleep.  That would mean that IPCs that do not block are not
 114 // cancelled or aborted.
 115 //-
 116
 117 #include <cstdlib>              // panic()
 118
 119 #include "l4_types.h"
 120 #include "l4_msg_item.h"
 121
 122 #include "config.h"
 123 #include "cpu_lock.h"
 124 #include "ipc_timeout.h"
 125 #include "lock_guard.h"
 126 #include "logdefs.h"
 127 #include "map_util.h"
 128 #include "processor.h"
 129 #include "timer.h"
 130 #include "warn.h"
 131
 132 PUBLIC
 133 virtual void
 134 Thread::ipc_receiver_aborted()
 135 {
 136   assert (wait_queue());
 137   set_wait_queue(0);
 138
 139   activate();
 140 }
 141
 142 PRIVATE
 143 void
 144 Thread::ipc_send_msg(Receiver *recv)
 145 {
 146   Syscall_frame *regs = _snd_regs;
 147   bool success = transfer_msg(regs->tag(), nonull_static_cast<Thread*>(recv), regs,
 148                               _ipc_send_rights);
 149   sender_dequeue(recv->sender_list());
 150   recv->vcpu_update_state();
 151   //printf("  done\n");
 152   regs->tag(L4_msg_tag(regs->tag(), success ? 0 : L4_msg_tag::Error));
 153
 154   Mword state_del = Thread_ipc_mask | Thread_ipc_transfer;
 155   Mword state_add = Thread_ready;
 156   if (Receiver::prepared())
 157     // same as in Receiver::prepare_receive_dirty_2
 158     state_add |= Thread_receive_wait;
 159
 160   if (xcpu_state_change(~state_del, state_add, true))
 161     recv->switch_to_locked(this);
 162 }
 163
 164 PUBLIC virtual
 165 void
 166 Thread::modify_label(Mword const *todo, int cnt)
 167 {
 168   assert (_snd_regs);
 169   Mword l = _snd_regs->from_spec();
 170   for (int i = 0; i < cnt*4; i += 4)
 171     {
 172       Mword const test_mask = todo[i];
 173       Mword const test      = todo[i+1];
 174       if ((l & test_mask) == test)
 175         {
 176           Mword const del_mask = todo[i+2];
 177           Mword const add_mask = todo[i+3];
 178
 179           l = (l & ~del_mask) | add_mask;
 180           _snd_regs->from(l);
 181           return;
 182         }
 183     }
 184 }
 185
 186 PRIVATE inline
 187 void
 188 Thread::snd_regs(Syscall_frame *r)
 189 { _snd_regs = r; }
 190
 191
 192 /** Page fault handler.
 193     This handler suspends any ongoing IPC, then sets up page-fault IPC.
 194     Finally, the ongoing IPC's state (if any) is restored.
 195     @param pfa page-fault virtual address
 196     @param error_code page-fault error code.
 197  */
 198 PRIVATE
 199 bool
 200 Thread::handle_page_fault_pager(Thread_ptr const &_pager,
 201                                 Address pfa, Mword error_code,
 202                                 L4_msg_tag::Protocol protocol)
 203 {
 204   if (EXPECT_FALSE((state() & Thread_alien)))
 205     return false;
 206
 207   auto guard = lock_guard(cpu_lock);
 208
 209   L4_fpage::Rights rights;
 210   Kobject_iface *pager = _pager.ptr(space(), &rights);
 211
 212   if (!pager)
 213     {
 214       WARN("CPU%u: Pager of %lx is invalid (pfa=" L4_PTR_FMT
 215            ", errorcode=" L4_PTR_FMT ") to %lx (pc=%lx)\n",
 216            cxx::int_value<Cpu_number>(current_cpu()), dbg_id(), pfa,
 217            error_code, cxx::int_value<Cap_index>(_pager.raw()), regs()->ip());
 218
 219
 220       LOG_TRACE("Page fault invalid pager", "ipfh", this, Log_pf_invalid,
 221                 l->cap_idx = _pager.raw();
 222                 l->err     = error_code;
 223                 l->pfa     = pfa);
 224
 225       pager = this; // block on ourselves
 226     }
 227
 228   // set up a register block used as an IPC parameter block for the
 229   // page fault IPC
 230   Syscall_frame r;
 231
 232   // save the UTCB fields affected by PF IPC
 233   Mword vcpu_irqs = vcpu_disable_irqs();
 234   Mem::barrier();
 235   Utcb *utcb = this->utcb().access(true);
 236   Pf_msg_utcb_saver saved_utcb_fields(utcb);
 237
 238
 239   utcb->buf_desc = L4_buf_desc(0, 0, 0, L4_buf_desc::Inherit_fpu);
 240   utcb->buffers[0] = L4_msg_item::map(0).raw();
 241   utcb->buffers[1] = L4_fpage::all_spaces().raw();
 242
 243   utcb->values[0] = PF::addr_to_msgword0(pfa, error_code);
 244   utcb->values[1] = regs()->ip(); //PF::pc_to_msgword1 (regs()->ip(), error_code));
 245
 246   L4_timeout_pair timeout(L4_timeout::Never, L4_timeout::Never);
 247
 248   L4_msg_tag tag(2, 0, 0, protocol);
 249
 250   r.timeout(timeout);
 251   r.tag(tag);
 252   r.from(0);
 253   r.ref(L4_obj_ref(_pager.raw(), L4_obj_ref::Ipc_call_ipc));
 254   pager->invoke(r.ref(), rights, &r, utcb);
 255
 256
 257   bool success = true;
 258
 259   if (EXPECT_FALSE(r.tag().has_error()))
 260     {
 261       if (utcb->error.snd_phase()
 262           && (utcb->error.error() == L4_error::Not_existent)
 263           && PF::is_usermode_error(error_code)
 264           && !(state() & Thread_cancel))
 265         {
 266           success = false;
 267         }
 268     }
 269   else // no error
 270     {
 271       // If the pager rejects the mapping, it replies -1 in msg.w0
 272       if (EXPECT_FALSE (utcb->values[0] == Mword(-1)))
 273         success = false;
 274     }
 275
 276   // restore previous IPC state
 277
 278   saved_utcb_fields.restore(utcb);
 279   Mem::barrier();
 280   vcpu_restore_irqs(vcpu_irqs);
 281   return success;
 282 }
 283
 284 PRIVATE inline
 285 Thread::Check_sender
 286 Thread::check_sender(Thread *sender, bool timeout)
 287 {
 288   if (EXPECT_FALSE(is_invalid()))
 289     {
 290       sender->utcb().access()->error = L4_error::Not_existent;
 291       return Check_sender::Failed;
 292     }
 293
 294   if (EXPECT_FALSE(!sender_ok(sender)))
 295     {
 296       if (!timeout)
 297         {
 298           sender->utcb().access()->error = L4_error::Timeout;
 299           return Check_sender::Failed;
 300         }
 301
 302       sender->set_wait_queue(sender_list());
 303       sender->sender_enqueue(sender_list(), sender->sched_context()->prio());
 304       vcpu_set_irq_pending();
 305       return Check_sender::Queued;
 306     }
 307
 308   return Check_sender::Ok;
 309 }
 310
 311 /**
 312  * Setup a IPC-like timer for the given timeout.
 313  * \param timeout  The L4 ABI timeout value that shall be used
 314  * \param utcb     The UTCB that might contain an absolute timeout
 315  * \param timer    The timeout/timer object that shall be queued.
 316  *
 317  * This function does nothing if the timeout is *never*.
 318  * Sets Thread_ready and Thread_timeout in the thread state
 319  * if the timeout is zero or has already hit (is in the past).
 320  * Or enqueues the given timer object with the finite timeout calculated
 321  * from `timeout`.
 322  */
 323 PUBLIC inline NEEDS["timer.h"]
 324 void
 325 Thread::setup_timer(L4_timeout timeout, Utcb const *utcb, Timeout *timer)
 326 {
 327   if (EXPECT_TRUE(timeout.is_never()))
 328     return;
 329
 330   if (EXPECT_FALSE(timeout.is_zero()))
 331     {
 332       state_add_dirty(Thread_ready | Thread_timeout);
 333       return;
 334     }
 335
 336   assert (!_timeout);
 337
 338   Unsigned64 sysclock = Timer::system_clock();
 339   Unsigned64 tval = timeout.microsecs(sysclock, utcb);
 340
 341   if (EXPECT_TRUE((tval > sysclock)))
 342     set_timeout(timer, tval);
 343   else // timeout already hit
 344     state_add_dirty(Thread_ready | Thread_timeout);
 345 }
 346
 347
 348 PRIVATE inline NEEDS["timer.h"]
 349 void Thread::goto_sleep(L4_timeout const &t, Sender *sender, Utcb *utcb)
 350 {
 351   IPC_timeout timeout;
 352
 353   state_del_dirty(Thread_ready);
 354   setup_timer(t, utcb, &timeout);
 355
 356   if (sender == this)
 357     switch_sched(sched(), &Sched_context::rq.current());
 358
 359   schedule();
 360
 361   reset_timeout();
 362
 363   assert (state() & Thread_ready);
 364 }
 365
 366
 367
 368 /**
 369  * @pre cpu_lock must be held
 370  */
 371 PRIVATE inline NEEDS["logdefs.h"]
 372 Thread::Check_sender
 373 Thread::handshake_receiver(Thread *partner, L4_timeout snd_t)
 374 {
 375   assert(cpu_lock.test());
 376
 377   switch (expect(partner->check_sender(this, !snd_t.is_zero()), Check_sender::Ok))
 378     {
 379     case Check_sender::Failed:
 380       return Check_sender::Failed;
 381     case Check_sender::Queued:
 382       state_add_dirty(Thread_send_wait);
 383       return Check_sender::Queued;
 384     default:
 385       partner->state_change_dirty(~(Thread_ipc_mask | Thread_ready), Thread_ipc_transfer);
 386       return Check_sender::Ok;
 387     }
 388 }
 389
 390 PRIVATE inline
 391 void
 392 Thread::set_ipc_error(L4_error const &e, Thread *rcv)
 393 {
 394   utcb().access()->error = e;
 395   rcv->utcb().access()->error = L4_error(e, L4_error::Rcv);
 396 }
 397
 398
 399 PRIVATE inline
 400 Sender *
 401 Thread::get_next_sender(Sender *sender)
 402 {
 403   if (!sender_list()->empty())
 404     {
 405       if (sender) // closed wait
 406         {
 407           if (EXPECT_TRUE(sender->in_sender_list())
 408               && EXPECT_TRUE(sender_list() == sender->wait_queue()))
 409             return sender;
 410           return 0;
 411         }
 412       else // open wait
 413         {
 414           Sender *next = Sender::cast(sender_list()->first());
 415           assert (next->in_sender_list());
 416           set_partner(next);
 417           return next;
 418         }
 419     }
 420   return 0;
 421 }
 422
 423 PRIVATE inline
 424 bool
 425 Thread::activate_ipc_partner(Thread *partner, Cpu_number current_cpu,
 426                              bool do_switch, bool closed_wait)
 427 {
 428   if (partner->home_cpu() == current_cpu)
 429     {
 430       auto &rq = Sched_context::rq.current();
 431       Sched_context *cs = rq.current_sched();
 432       do_switch = do_switch && (closed_wait || cs != sched());
 433       partner->state_change_dirty(~Thread_ipc_transfer, Thread_ready);
 434       if (do_switch)
 435         {
 436           schedule_if(switch_exec_locked(partner, Not_Helping) != Switch::Ok);
 437           return true;
 438         }
 439       else
 440         return deblock_and_schedule(partner);
 441     }
 442
 443   partner->xcpu_state_change(~Thread_ipc_transfer, Thread_ready);
 444   return false;
 445 }
 446
 447 /**
 448  * Send an IPC message.
 449  *        Block until we can send the message or the timeout hits.
 450  * @param partner the receiver of our message
 451  * @param t a timeout specifier
 452  * @param regs sender's IPC registers
 453  * @pre cpu_lock must be held
 454  * @return sender's IPC error code
 455  *
 456  * @todo review closed wait handling of sender during possible
 457  *       quiescent states and blocking.
 458  */
 459 PUBLIC
 460 void
 461 Thread::do_ipc(L4_msg_tag const &tag, bool have_send, Thread *partner,
 462                bool have_receive, Sender *sender,
 463                L4_timeout_pair t, Syscall_frame *regs,
 464                L4_fpage::Rights rights)
 465 {
 466   assert (cpu_lock.test());
 467   assert (this == current());
 468
 469   bool do_switch = false;
 470
 471   assert (!(state() & Thread_ipc_mask));
 472
 473   prepare_receive(sender, have_receive ? regs : 0);
 474   bool activate_partner = false;
 475   Cpu_number current_cpu = ::current_cpu();
 476
 477   if (have_send)
 478     {
 479       assert(!in_sender_list());
 480       do_switch = tag.do_switch();
 481
 482       bool ok;
 483       Check_sender result;
 484
 485       set_ipc_send_rights(rights);
 486
 487       if (EXPECT_TRUE(current_cpu == partner->home_cpu()))
 488         result = handshake_receiver(partner, t.snd);
 489       else
 490         {
 491           // we have either per se X-CPU IPC or we ran into a
 492           // IPC during migration (indicated by the pending DRQ)
 493           do_switch = false;
 494           result = remote_handshake_receiver(tag, partner, have_receive, t.snd,
 495                                              regs, rights);
 496
 497           // this may block, so we could have been migrated here
 498           current_cpu = ::current_cpu();
 499         }
 500
 501       switch (expect(result, Check_sender::Ok))
 502         {
 503         case Check_sender::Done:
 504           ok = true;
 505           break;
 506
 507         case Check_sender::Queued:
 508           // set _snd_regs, to enable active receiving
 509           snd_regs(regs);
 510           ok = do_send_wait(partner, t.snd); // --- blocking point ---
 511           current_cpu = ::current_cpu();
 512           break;
 513
 514         case Check_sender::Failed:
 515           state_del_dirty(Thread_ipc_mask);
 516           ok = false;
 517           break;
 518
 519         default:
 520           // mmh, we can reset the receivers timeout
 521           // ping pong with timeouts will profit from it, because
 522           // it will require much less sorting overhead
 523           // if we dont reset the timeout, the possibility is very high
 524           // that the receiver timeout is in the timeout queue
 525           if (EXPECT_TRUE(current_cpu == partner->home_cpu()))
 526             partner->reset_timeout();
 527
 528           ok = transfer_msg(tag, partner, regs, rights);
 529
 530           // transfer is also a possible migration point
 531           current_cpu = ::current_cpu();
 532
 533           // switch to receiving state
 534           state_del_dirty(Thread_ipc_mask);
 535           if (ok && have_receive)
 536             state_add_dirty(Thread_receive_wait);
 537
 538           activate_partner = partner != this;
 539           break;
 540         }
 541
 542       if (EXPECT_FALSE(!ok))
 543         {
 544           // send failed, so do not switch to receiver directly and skip receive phase
 545           have_receive = false;
 546           regs->tag(L4_msg_tag(0, 0, L4_msg_tag::Error, 0));
 547         }
 548     }
 549   else
 550     {
 551       assert (have_receive);
 552       state_add_dirty(Thread_receive_wait);
 553     }
 554
 555   // only do direct switch on closed wait (call) or if we run on a foreign
 556   // scheduling context
 557   Sender *next = 0;
 558
 559   have_receive = state() & Thread_receive_wait;
 560
 561   if (have_receive)
 562     {
 563       assert (!in_sender_list());
 564       assert (!(state() & Thread_send_wait));
 565       next = get_next_sender(sender);
 566     }
 567
 568   if (activate_partner
 569       && activate_ipc_partner(partner, current_cpu, do_switch && !next,
 570                               have_receive && sender))
 571     {
 572       // blocked so might have a new sender queued
 573       have_receive = state() & Thread_receive_wait;
 574       if (have_receive && !next)
 575         next = get_next_sender(sender);
 576     }
 577
 578   if (next)
 579     {
 580       state_change_dirty(~Thread_ipc_mask, Thread_receive_in_progress);
 581       next->ipc_send_msg(this);
 582       state_del_dirty(Thread_ipc_mask);
 583     }
 584   else if (have_receive)
 585     {
 586       if ((state() & Thread_full_ipc_mask) == Thread_receive_wait)
 587         goto_sleep(t.rcv, sender, utcb().access(true));
 588
 589       if (sender && sender == partner && partner->caller() == this)
 590         partner->reset_caller();
 591     }
 592
 593   Mword state = this->state();
 594
 595   if (EXPECT_TRUE (!(state & Thread_full_ipc_mask)))
 596     return;
 597
 598   while (EXPECT_FALSE(state & Thread_ipc_transfer))
 599     {
 600       state_del_dirty(Thread_ready);
 601       schedule();
 602       state = this->state();
 603    }
 604
 605   if (EXPECT_TRUE (!(state & Thread_full_ipc_mask)))
 606     return;
 607
 608   if (state & Thread_ipc_mask)
 609     {
 610       Utcb *utcb = this->utcb().access(true);
 611       // the IPC has not been finished.  could be timeout or cancel
 612       // XXX should only modify the error-code part of the status code
 613
 614       if (EXPECT_FALSE(state & Thread_cancel))
 615         {
 616           // we've presumably been reset!
 617           regs->tag(commit_error(utcb, L4_error::R_canceled, regs->tag()));
 618         }
 619       else
 620         regs->tag(commit_error(utcb, L4_error::R_timeout, regs->tag()));
 621     }
 622   state_del(Thread_full_ipc_mask);
 623 }
 624
 625
 626 PRIVATE inline NEEDS [Thread::copy_utcb_to]
 627 bool
 628 Thread::transfer_msg(L4_msg_tag tag, Thread *receiver,
 629                      Syscall_frame *sender_regs, L4_fpage::Rights rights)
 630 {
 631   Syscall_frame* dst_regs = receiver->rcv_regs();
 632
 633   bool success = copy_utcb_to(tag, receiver, rights);
 634   tag.set_error(!success);
 635   dst_regs->tag(tag);
 636   dst_regs->from(sender_regs->from_spec());
 637
 638   // setup the reply capability in case of a call
 639   if (success && partner() == receiver)
 640     receiver->set_caller(this, rights);
 641
 642   return success;
 643 }
 644
 645
 646
 647 IMPLEMENT inline
 648 Buf_utcb_saver::Buf_utcb_saver(const Utcb *u)
 649 {
 650   buf_desc = u->buf_desc;
 651   buf[0] = u->buffers[0];
 652   buf[1] = u->buffers[1];
 653 }
 654
 655 IMPLEMENT inline
 656 void
 657 Buf_utcb_saver::restore(Utcb *u)
 658 {
 659   u->buf_desc = buf_desc;
 660   u->buffers[0] = buf[0];
 661   u->buffers[1] = buf[1];
 662 }
 663
 664 IMPLEMENT inline
 665 Pf_msg_utcb_saver::Pf_msg_utcb_saver(Utcb const *u) : Buf_utcb_saver(u)
 666 {
 667   msg[0] = u->values[0];
 668   msg[1] = u->values[1];
 669 }
 670
 671 IMPLEMENT inline
 672 void
 673 Pf_msg_utcb_saver::restore(Utcb *u)
 674 {
 675   Buf_utcb_saver::restore(u);
 676   u->values[0] = msg[0];
 677   u->values[1] = msg[1];
 678 }
 679
 680
 681 /**
 682  * \pre must run with local IRQs disabled (CPU lock held)
 683  * to ensure that handler does not dissapear meanwhile.
 684  */
 685 PRIVATE
 686 bool
 687 Thread::exception(Kobject_iface *handler, Trap_state *ts, L4_fpage::Rights rights)
 688 {
 689   Syscall_frame r;
 690   L4_timeout_pair timeout(L4_timeout::Never, L4_timeout::Never);
 691
 692   CNT_EXC_IPC;
 693
 694   Mword vcpu_irqs = vcpu_disable_irqs();
 695   Mem::barrier();
 696
 697   void *old_utcb_handler = _utcb_handler;
 698   _utcb_handler = ts;
 699
 700   // fill registers for IPC
 701   Utcb *utcb = this->utcb().access(true);
 702   Buf_utcb_saver saved_state(utcb);
 703
 704   utcb->buf_desc = L4_buf_desc(0, 0, 0, L4_buf_desc::Inherit_fpu);
 705   utcb->buffers[0] = L4_msg_item::map(0).raw();
 706   utcb->buffers[1] = L4_fpage::all_spaces().raw();
 707
 708   // clear regs
 709   L4_msg_tag tag(L4_exception_ipc::Msg_size, 0, L4_msg_tag::Transfer_fpu,
 710                  L4_msg_tag::Label_exception);
 711
 712   r.tag(tag);
 713   r.timeout(timeout);
 714   r.from(0);
 715   r.ref(L4_obj_ref(_exc_handler.raw(), L4_obj_ref::Ipc_call_ipc));
 716   spill_user_state();
 717   handler->invoke(r.ref(), rights, &r, utcb);
 718   fill_user_state();
 719
 720   saved_state.restore(utcb);
 721
 722   if (EXPECT_FALSE(r.tag().has_error()))
 723     state_del(Thread_in_exception);
 724   else if (r.tag().proto() == L4_msg_tag::Label_allow_syscall)
 725     state_add(Thread_dis_alien);
 726
 727   // restore original utcb_handler
 728   _utcb_handler = old_utcb_handler;
 729   Mem::barrier();
 730   vcpu_restore_irqs(vcpu_irqs);
 731
 732   // FIXME: handle not existing pager properly
 733   // for now, just ignore any errors
 734   return 1;
 735 }
 736
 737 /* return 1 if exception could be handled
 738  * return 0 if not for send_exception and halt thread
 739  */
 740 PUBLIC inline NEEDS["task.h", "trap_state.h",
 741                     Thread::fast_return_to_user,
 742                     Thread::save_fpu_state_to_utcb]
 743 int
 744 Thread::send_exception(Trap_state *ts)
 745 {
 746   assert(cpu_lock.test());
 747
 748   Vcpu_state *vcpu = vcpu_state().access();
 749
 750   if (vcpu_exceptions_enabled(vcpu))
 751     {
 752       // do not reflect debug exceptions to the VCPU but handle them in
 753       // Fiasco
 754       if (EXPECT_FALSE(ts->is_debug_exception()
 755                        && !(vcpu->state & Vcpu_state::F_debug_exc)))
 756         return 0;
 757
 758       if (_exc_cont.valid(ts))
 759         return 1;
 760
 761       // before entering kernel mode to have original fpu state before
 762       // enabling fpu
 763       save_fpu_state_to_utcb(ts, utcb().access());
 764
 765       spill_user_state();
 766
 767       if (vcpu_enter_kernel_mode(vcpu))
 768         {
 769           // enter_kernel_mode has switched the address space from user to
 770           // kernel space, so reevaluate the address of the VCPU state area
 771           vcpu = vcpu_state().access();
 772         }
 773
 774       LOG_TRACE("VCPU events", "vcpu", this, Vcpu_log,
 775           l->type = 2;
 776           l->state = vcpu->_saved_state;
 777           l->ip = ts->ip();
 778           l->sp = ts->sp();
 779           l->trap = ts->trapno();
 780           l->err = ts->error();
 781           l->space = vcpu_user_space() ? static_cast<Task*>(vcpu_user_space())->dbg_id() : ~0;
 782           );
 783       vcpu->_regs.s = *ts;
 784       fast_return_to_user(vcpu->_entry_ip, vcpu->_sp, vcpu_state().usr().get());
 785     }
 786
 787   // local IRQs must be disabled because we dereference a Thread_ptr
 788   if (EXPECT_FALSE(_exc_handler.is_kernel()))
 789     return 0;
 790
 791   if (!send_exception_arch(ts))
 792     return 0; // do not send exception
 793
 794   L4_fpage::Rights rights = L4_fpage::Rights(0);
 795   Kobject_iface *pager = _exc_handler.ptr(space(), &rights);
 796
 797   if (EXPECT_FALSE(!pager))
 798     {
 799       /* no pager (anymore), just ignore the exception, return success */
 800       LOG_TRACE("Exception invalid handler", "ieh", this, Log_exc_invalid,
 801                 l->cap_idx = _exc_handler.raw());
 802       if (EXPECT_FALSE(space()->is_sigma0()))
 803         {
 804           ts->dump();
 805           WARNX(Error, "Sigma0 raised an exception --> HALT\n");
 806           panic("...");
 807         }
 808
 809       pager = this; // block on ourselves
 810     }
 811
 812   state_change(~Thread_cancel, Thread_in_exception);
 813
 814   return exception(pager, ts, rights);
 815 }
 816
 817 PRIVATE static
 818 bool
 819 Thread::try_transfer_local_id(L4_buf_iter::Item const *const buf,
 820                               L4_fpage sfp, Mword *rcv_word, Thread* snd,
 821                               Thread *rcv)
 822 {
 823   if (buf->b.is_rcv_id())
 824     {
 825       if (snd->space() == rcv->space())
 826         {
 827           rcv_word[-2] |= 6;
 828           rcv_word[-1] = sfp.raw();
 829           return true;
 830         }
 831       else
 832         {
 833           Obj_space::Capability cap = snd->space()->lookup(sfp.obj_index());
 834           Kobject_iface *o = cap.obj();
 835           if (EXPECT_TRUE(o && o->is_local(rcv->space())))
 836             {
 837               Mword rights = cap.rights()
 838                              & cxx::int_value<L4_fpage::Rights>(sfp.rights());
 839               rcv_word[-2] |= 4;
 840               rcv_word[-1] = o->obj_id() | rights;
 841               return true;
 842             }
 843         }
 844     }
 845   return false;
 846 }
 847
 848 PRIVATE static inline
 849 bool FIASCO_WARN_RESULT
 850 Thread::copy_utcb_to_utcb(L4_msg_tag const &tag, Thread *snd, Thread *rcv,
 851                           L4_fpage::Rights rights)
 852 {
 853   assert (cpu_lock.test());
 854
 855   Utcb *snd_utcb = snd->utcb().access();
 856   Utcb *rcv_utcb = rcv->utcb().access();
 857   Mword s = tag.words();
 858   Mword r = Utcb::Max_words;
 859
 860   Mem::memcpy_mwords(rcv_utcb->values, snd_utcb->values, r < s ? r : s);
 861
 862   bool success = true;
 863   if (tag.items())
 864     success = transfer_msg_items(tag, snd, snd_utcb, rcv, rcv_utcb, rights);
 865
 866   if (success
 867       && tag.transfer_fpu()
 868       && rcv_utcb->inherit_fpu()
 869       && (rights & L4_fpage::Rights::W()))
 870     snd->transfer_fpu(rcv);
 871
 872   return success;
 873 }
 874
 875
 876 PUBLIC inline NEEDS[Thread::copy_utcb_to_ts, Thread::copy_utcb_to_utcb,
 877                     Thread::copy_ts_to_utcb]
 878 bool FIASCO_WARN_RESULT
 879 Thread::copy_utcb_to(L4_msg_tag tag, Thread* receiver,
 880                      L4_fpage::Rights rights)
 881 {
 882   // we cannot copy trap state to trap state!
 883   assert (!this->_utcb_handler || !receiver->_utcb_handler);
 884   if (EXPECT_FALSE(this->_utcb_handler != 0))
 885     return copy_ts_to_utcb(tag, this, receiver, rights);
 886   else if (EXPECT_FALSE(receiver->_utcb_handler != 0))
 887     return copy_utcb_to_ts(tag, this, receiver, rights);
 888   else
 889     return copy_utcb_to_utcb(tag, this, receiver, rights);
 890 }
 891
 892 PRIVATE static
 893 bool
 894 Thread::transfer_msg_items(L4_msg_tag const &tag, Thread* snd, Utcb *snd_utcb,
 895                            Thread *rcv, Utcb *rcv_utcb,
 896                            L4_fpage::Rights rights)
 897 {
 898   // LOG_MSG_3VAL(current(), "map bd=", rcv_utcb->buf_desc.raw(), 0, 0);
 899   Task *const rcv_t = nonull_static_cast<Task*>(rcv->space());
 900   L4_buf_iter mem_buffer(rcv_utcb, rcv_utcb->buf_desc.mem());
 901   L4_buf_iter io_buffer(rcv_utcb, rcv_utcb->buf_desc.io());
 902   L4_buf_iter obj_buffer(rcv_utcb, rcv_utcb->buf_desc.obj());
 903   L4_snd_item_iter snd_item(snd_utcb, tag.words());
 904   int items = tag.items();
 905   Mword *rcv_word = rcv_utcb->values + tag.words();
 906
 907   // XXX: damn X-CPU state modification
 908   // snd->prepare_long_ipc(rcv);
 909   Reap_list rl;
 910
 911   for (;items > 0 && snd_item.more();)
 912     {
 913       if (EXPECT_FALSE(!snd_item.next()))
 914         {
 915           snd->set_ipc_error(L4_error::Overflow, rcv);
 916           return false;
 917         }
 918
 919       L4_snd_item_iter::Item const *const item = snd_item.get();
 920
 921       if (item->b.is_void())
 922         { // XXX: not sure if void fpages are needed
 923           // skip send item and current rcv_buffer
 924           --items;
 925           continue;
 926         }
 927
 928       L4_buf_iter *buf_iter = 0;
 929
 930       switch (item->b.type())
 931         {
 932         case L4_msg_item::Map:
 933           switch (L4_fpage(item->d).type())
 934             {
 935             case L4_fpage::Memory: buf_iter = &mem_buffer; break;
 936             case L4_fpage::Io:     buf_iter = &io_buffer; break;
 937             case L4_fpage::Obj:    buf_iter = &obj_buffer; break;
 938             default: break;
 939             }
 940           break;
 941         default:
 942           break;
 943         }
 944
 945       if (EXPECT_FALSE(!buf_iter))
 946         {
 947           // LOG_MSG_3VAL(snd, "lIPCm0", 0, 0, 0);
 948           snd->set_ipc_error(L4_error::Overflow, rcv);
 949           return false;
 950         }
 951
 952       L4_buf_iter::Item const *const buf = buf_iter->get();
 953
 954       if (EXPECT_FALSE(buf->b.is_void() || buf->b.type() != item->b.type()))
 955         {
 956           // LOG_MSG_3VAL(snd, "lIPCm1", buf->b.raw(), item->b.raw(), 0);
 957           snd->set_ipc_error(L4_error::Overflow, rcv);
 958           return false;
 959         }
 960
 961         {
 962           assert (item->b.type() == L4_msg_item::Map);
 963           L4_fpage sfp(item->d);
 964           *rcv_word = (item->b.raw() & ~0x0ff7) | (sfp.raw() & 0x0ff0);
 965
 966           rcv_word += 2;
 967
 968           // diminish when sending via restricted ipc gates
 969           if (sfp.type() == L4_fpage::Obj)
 970             sfp.mask_rights(rights | L4_fpage::Rights::CRW() | L4_fpage::Rights::CD());
 971
 972           if (!try_transfer_local_id(buf, sfp, rcv_word, snd, rcv))
 973             {
 974               // we need to do a real mapping
 975               L4_error err;
 976
 977                 {
 978                   // We take the existence_lock for syncronizing maps...
 979                   // This is kind of coarse grained
 980                   auto sp_lock = lock_guard_dont_lock(rcv_t->existence_lock);
 981                   if (!sp_lock.check_and_lock(&rcv_t->existence_lock))
 982                     {
 983                       snd->set_ipc_error(L4_error::Overflow, rcv);
 984                       return false;
 985                     }
 986
 987                   auto c_lock = lock_guard<Lock_guard_inverse_policy>(cpu_lock);
 988                   err = fpage_map(snd->space(), sfp,
 989                                   rcv->space(), L4_fpage(buf->d), item->b, &rl);
 990                 }
 991
 992               if (EXPECT_FALSE(!err.ok()))
 993                 {
 994                   snd->set_ipc_error(err, rcv);
 995                   return false;
 996                 }
 997             }
 998         }
 999
1000       --items;
1001
1002       if (!item->b.compound())
1003         buf_iter->next();
1004     }
1005
1006   if (EXPECT_FALSE(items))
1007     {
1008       snd->set_ipc_error(L4_error::Overflow, rcv);
1009       return false;
1010     }
1011
1012   return true;
1013 }
1014
1015
1016 /**
1017  * \pre Runs on the sender CPU
1018  */
1019 PRIVATE inline
1020 bool
1021 Thread::abort_send(L4_error const &e, Thread *partner)
1022 {
1023   state_del_dirty(Thread_full_ipc_mask);
1024
1025   if (_timeout && _timeout->is_set())
1026     _timeout->reset();
1027
1028   set_timeout(0);
1029   Abort_state abt = Abt_ipc_done;
1030
1031   if (partner->home_cpu() == current_cpu())
1032     {
1033       if (in_sender_list())
1034         {
1035           sender_dequeue(partner->sender_list());
1036           partner->vcpu_update_state();
1037           abt = Abt_ipc_cancel;
1038         }
1039       else if (partner->in_ipc(this))
1040         abt = Abt_ipc_in_progress;
1041     }
1042   else
1043     abt = partner->Receiver::abort_send(this);
1044
1045   switch (abt)
1046     {
1047     default:
1048     case Abt_ipc_done:
1049       return true;
1050     case Abt_ipc_cancel:
1051       utcb().access()->error = e;
1052       return false;
1053     case Abt_ipc_in_progress:
1054       state_add_dirty(Thread_ipc_transfer);
1055       while (state() & Thread_ipc_transfer)
1056         {
1057           state_del_dirty(Thread_ready);
1058           schedule();
1059         }
1060       return true;
1061     }
1062 }
1063
1064
1065
1066 /**
1067  * \pre Runs on the sender CPU
1068  */
1069 PRIVATE inline
1070 bool
1071 Thread::do_send_wait(Thread *partner, L4_timeout snd_t)
1072 {
1073   IPC_timeout timeout;
1074
1075   if (EXPECT_FALSE(snd_t.is_finite()))
1076     {
1077       Unsigned64 tval = snd_t.microsecs(Timer::system_clock(), utcb().access(true));
1078       // Zero timeout or timeout expired already -- give up
1079       if (tval == 0)
1080         return abort_send(L4_error::Timeout, partner);
1081
1082       set_timeout(&timeout, tval);
1083     }
1084
1085   Mword ipc_state;
1086
1087   while (((ipc_state = state() & (Thread_send_wait | Thread_ipc_abort_mask))) == Thread_send_wait)
1088     {
1089       state_del_dirty(Thread_ready);
1090       schedule();
1091     }
1092
1093   if (EXPECT_FALSE(ipc_state == (Thread_cancel | Thread_send_wait)))
1094     return abort_send(L4_error::Canceled, partner);
1095
1096   if (EXPECT_FALSE(ipc_state == (Thread_timeout | Thread_send_wait)))
1097     return abort_send(L4_error::Timeout, partner);
1098
1099   reset_timeout();
1100
1101   return true;
1102 }
1103
1104 PRIVATE inline
1105 void
1106 Thread::set_ipc_send_rights(L4_fpage::Rights c)
1107 {
1108   _ipc_send_rights = c;
1109 }
1110
1111 PRIVATE inline NOEXPORT
1112 bool
1113 Thread::remote_ipc_send(Ipc_remote_request *rq)
1114 {
1115
1116 #if 0
1117   LOG_MSG_3VAL(this, "rsend", (Mword)src, 0, 0);
1118   printf("CPU[%u]: remote IPC send ...\n"
1119          "  partner=%p [%u]\n"
1120          "  sender =%p [%u] regs=%p\n"
1121          "  timeout=%u\n",
1122          current_cpu(),
1123          rq->partner, rq->partner->cpu(),
1124          src, src->cpu(),
1125          rq->regs,
1126          rq->timeout);
1127 #endif
1128
1129   switch (expect(rq->partner->check_sender(this, rq->timeout), Check_sender::Ok))
1130     {
1131     case Check_sender::Failed:
1132       xcpu_state_change(~Thread_ipc_mask, 0);
1133       rq->result = Check_sender::Failed;
1134       return false;
1135     case Check_sender::Queued:
1136       rq->result = Check_sender::Queued;
1137       return false;
1138     default:
1139       break;
1140     }
1141
1142   if (rq->tag.transfer_fpu() && rq->partner->_utcb_handler || rq->partner->utcb().access()->inherit_fpu())
1143     rq->partner->spill_fpu_if_owner();
1144
1145   // trigger remote_ipc_receiver_ready path, because we may need to grab locks
1146   // and this is forbidden in a DRQ handler. So transfer the IPC in usual
1147   // thread code. However, this induces a overhead of two extra IPIs.
1148   if (rq->tag.items())
1149     {
1150       //LOG_MSG_3VAL(rq->partner, "pull", dbg_id(), 0, 0);
1151       xcpu_state_change(~Thread_send_wait, Thread_ready);
1152       rq->partner->state_change_dirty(~(Thread_ipc_mask | Thread_ready), Thread_ipc_transfer);
1153       rq->result = Check_sender::Ok;
1154       return true;
1155     }
1156   bool success = transfer_msg(rq->tag, rq->partner, rq->regs, _ipc_send_rights);
1157   if (success && rq->have_rcv)
1158     xcpu_state_change(~Thread_send_wait, Thread_receive_wait);
1159   else
1160     xcpu_state_change(~Thread_ipc_mask, 0);
1161
1162   rq->result = success ? Check_sender::Done : Check_sender::Failed;
1163   rq->partner->state_change_dirty(~Thread_ipc_mask, Thread_ready);
1164   if (rq->partner->home_cpu() == current_cpu() && current() != rq->partner)
1165     Sched_context::rq.current().ready_enqueue(rq->partner->sched());
1166
1167   return true;
1168 }
1169
1170 PRIVATE static
1171 Context::Drq::Result
1172 Thread::handle_remote_ipc_send(Drq *src, Context *, void *_rq)
1173 {
1174   Ipc_remote_request *rq = (Ipc_remote_request*)_rq;
1175   bool r = nonull_static_cast<Thread*>(src->context())->remote_ipc_send(rq);
1176   //LOG_MSG_3VAL(src, "rse<", current_cpu(), (Mword)src, r);
1177   return r ? Drq::need_resched() : Drq::done();
1178 }
1179
1180 /**
1181  * \pre Runs on the sender CPU
1182  */
1183 PRIVATE //inline NEEDS ["mp_request.h"]
1184 Thread::Check_sender
1185 Thread::remote_handshake_receiver(L4_msg_tag const &tag, Thread *partner,
1186                                   bool have_receive,
1187                                   L4_timeout snd_t, Syscall_frame *regs,
1188                                   L4_fpage::Rights rights)
1189 {
1190   // Flag that there must be no switch in the receive path.
1191   // This flag also prevents the receive path from accessing
1192   // the thread state of a remote sender.
1193   Ipc_remote_request rq;
1194   rq.tag = tag;
1195   rq.have_rcv = have_receive;
1196   rq.partner = partner;
1197   rq.timeout = !snd_t.is_zero();
1198   rq.regs = regs;
1199   rq.rights = rights;
1200   _snd_regs = regs;
1201
1202   set_wait_queue(partner->sender_list());
1203
1204   state_add_dirty(Thread_send_wait);
1205
1206   if (tag.transfer_fpu())
1207     spill_fpu_if_owner();
1208
1209   partner->drq(handle_remote_ipc_send, &rq);
1210
1211   return rq.result;
1212 }
1213
1214 //---------------------------------------------------------------------------
1215 IMPLEMENTATION [debug]:
1216
1217 #include "string_buffer.h"
1218
1219 IMPLEMENT
1220 void
1221 Thread::Log_pf_invalid::print(String_buffer *buf) const
1222 {
1223   buf->printf("InvCap C:%lx pfa=%lx err=%lx",
1224               cxx::int_value<Cap_index>(cap_idx), pfa, err);
1225 }
1226
1227 IMPLEMENT
1228 void
1229 Thread::Log_exc_invalid::print(String_buffer *buf) const
1230 {
1231   buf->printf("InvCap C:%lx", cxx::int_value<Cap_index>(cap_idx));
1232 }