12 // ------------------------------------------------------------------------
19 #include "kmem_space.h"
21 #include "static_assert.h"
22 #include "thread_state.h"
24 #include "vmem_alloc.h"
27 FSR_STATUS_MASK = 0x0d,
30 FSR_PERMISSION = 0x0d,
33 Per_cpu<Thread::Dbg_stack> DEFINE_PER_CPU Thread::dbg_stack;
37 Thread::print_page_fault_error(Mword e)
39 char const *const excpts[] =
40 { "reset","undef. insn", "swi", "pref. abort", "data abort",
41 "XXX", "XXX", "XXX" };
43 unsigned ex = (e >> 20) & 0x07;
45 printf("(%lx) %s, %s(%c)",e & 0xff, excpts[ex],
46 (e & 0x00010000)?"user":"kernel",
47 (e & 0x00020000)?'r':'w');
52 Thread::fast_return_to_user(Mword ip, Mword sp, Vcpu_state *arg)
55 assert_kdb((regs()->psr & Proc::Status_mode_mask) == Proc::Status_mode_user);
58 regs()->sp(sp); // user-sp is in lazy user state and thus handled by
62 regs()->psr &= ~Proc::Status_thumb;
65 register Vcpu_state *r0 asm("r0") = arg;
71 : "r" (nonull_static_cast<Return_frame*>(regs())), "r" (__iret), "r"(r0)
74 panic("__builtin_trap()");
85 user_invoke_generic();
86 assert (current()->state() & Thread_ready);
88 Trap_state *ts = nonull_static_cast<Trap_state*>
89 (nonull_static_cast<Return_frame*>(current()->regs()));
91 static_assert(sizeof(ts->r[0]), sizeof(Mword));
92 Mem::memset_mwords(&ts->r[0], 0, sizeof(ts->r) / sizeof(ts->r[0]));
94 if (current()->space() == sigma0_task)
95 ts->r[0] = Kmem_space::kdir()->walk(Kip::k(), 0, false, 0).phys(Kip::k());
97 extern char __return_from_exception;
100 (" mov sp, %[stack_p] \n" // set stack pointer to regs structure
105 [rfe] "r" (&__return_from_exception)
108 panic("should never be reached");
111 current()->state_del(Thread_ready);
112 current()->schedule();
115 // never returns here
118 IMPLEMENT inline NEEDS["space.h", <cstdio>, "types.h" ,"config.h"]
119 bool Thread::handle_sigma0_page_fault( Address pfa )
121 return (mem_space()->v_insert(
122 Mem_space::Phys_addr::create((pfa & Config::SUPERPAGE_MASK)),
123 Mem_space::Addr::create(pfa & Config::SUPERPAGE_MASK),
124 Mem_space::Size(Config::SUPERPAGE_SIZE),
125 Mem_space::Page_writable | Mem_space::Page_user_accessible
126 | Mem_space::Page_cacheable)
127 != Mem_space::Insert_err_nomem);
134 * The low-level page fault handler called from entry.S. We're invoked with
135 * interrupts turned off. Apart from turning on interrupts in almost
136 * all cases (except for kernel page faults in TCB area), just forwards
137 * the call to Thread::handle_page_fault().
138 * @param pfa page-fault virtual address
139 * @param error_code CPU error code
140 * @return true if page fault could be resolved, false otherwise
142 Mword pagefault_entry(const Mword pfa, Mword error_code,
143 const Mword pc, Return_frame *ret_frame)
145 #if 0 // Double PF detect
146 static unsigned long last_pfa = ~0UL;
147 LOG_MSG_3VAL(current(),"PF", pfa, error_code, pc);
148 if (last_pfa == pfa || pfa == 0)
152 if (EXPECT_FALSE(PF::is_alignment_error(error_code)))
154 printf("KERNEL%d: alignment error at %08lx (PC: %08lx, SP: %08lx, FSR: %lx, PSR: %lx)\n",
155 current_cpu(), pfa, pc, ret_frame->usp, error_code, ret_frame->psr);
159 Thread *t = current_thread();
161 // Pagefault in user mode
162 if (PF::is_usermode_error(error_code))
164 if (t->vcpu_pagefault(pfa, error_code, pc))
166 t->state_del(Thread_cancel);
169 // or interrupts were enabled
170 else if (!(ret_frame->psr & Proc::Status_IRQ_disabled))
173 // Pagefault in kernel mode and interrupts were disabled
176 // page fault in kernel memory region, not present, but mapping exists
177 if (Kmem::is_kmem_page_fault (pfa, error_code))
179 // We've interrupted a context in the kernel with disabled interrupts,
180 // the page fault address is in the kernel region, the error code is
181 // "not mapped" (as opposed to "access error"), and the region is
182 // actually valid (that is, mapped in Kmem's shared page directory,
183 // just not in the currently active page directory)
186 else if (!Config::conservative &&
187 !Kmem::is_kmem_page_fault (pfa, error_code))
189 // No error -- just enable interrupts.
194 // Error: We interrupted a cli'd kernel context touching kernel space
195 if (!Thread::log_page_fault())
196 printf("*P[%lx,%lx,%lx] ", pfa, error_code, pc);
198 kdb_ke ("page fault in cli mode");
203 // cache operations we carry out for user space might cause PFs, we just
205 if (EXPECT_FALSE(t->is_ignore_mem_op_in_progress()))
211 // PFs in the kern_lib_page are always write PFs due to rollbacks and
213 if (EXPECT_FALSE((pc & Kmem::Kern_lib_base) == Kmem::Kern_lib_base))
214 error_code |= (1UL << 11);
216 return t->handle_page_fault(pfa, error_code, pc, ret_frame);
219 void slowtrap_entry(Trap_state *ts)
221 Thread *t = current_thread();
225 if (Config::Support_arm_linux_cache_API)
227 if ( ts->error_code == 0x00200000
228 && ts->r[7] == 0xf0002)
231 Mem_op::arm_mem_cache_maint(Mem_op::Op_cache_coherent,
232 (void *)ts->r[0], (void *)ts->r[1]);
237 if (ts->exception_is_undef_insn())
239 switch (Fpu::handle_fpu_trap(ts))
241 case Fpu::Fpu_except_emulated: return;
242 case Fpu::Fpu_except_fault:
244 if (!(current_thread()->state() & Thread_vcpu_enabled)
245 && Fpu::is_enabled() && Fpu::owner(t->cpu()) == t)
246 printf("KERNEL: FPU doesn't like us?\n");
249 if (t->switchin_fpu())
251 ts->pc -= (ts->psr & Proc::Status_thumb) ? 2 : 4;
255 ts->error_code |= 0x01000000; // tag fpu undef insn
256 if (Fpu::exc_pending())
257 ts->error_code |= 0x02000000; // fpinst and fpinst2 in utcb will be valid
259 case Fpu::Fpu_except_none: break;
263 // send exception IPC if requested
264 if (t->send_exception(ts))
267 // exception handling failed
268 if (Config::conservative)
269 kdb_ke ("thread killed");
279 Thread::pagein_tcb_request(Return_frame *regs)
281 //if ((*(Mword*)regs->pc & 0xfff00fff ) == 0xe5900000)
282 if (*(Mword*)regs->pc == 0xe59ee000)
284 // printf("TCBR: %08lx\n", *(Mword*)regs->pc);
285 // skip faulting instruction
287 // tell program that a pagefault occured we cannot handle
288 regs->psr |= 0x40000000; // set zero flag in psr
296 //---------------------------------------------------------------------------
297 IMPLEMENTATION [arm]:
299 #include "trap_state.h"
303 @param space the address space
304 @param id user-visible thread ID of the sender
305 @param init_prio initial priority
306 @param mcp thread's maximum controlled priority
307 @post state() != Thread_invalid
311 : Sender(0), // select optimized version of constructor
312 _pager(Thread_ptr::Invalid),
313 _exc_handler(Thread_ptr::Invalid),
316 assert (state() == Thread_invalid);
320 if (Config::stack_depth)
321 std::memset((char*)this + sizeof(Thread), '5',
322 Config::thread_block_size-sizeof(Thread)-64);
324 // set a magic value -- we use it later to verify the stack hasn't
329 _in_exception = false;
331 *reinterpret_cast<void(**)()> (--_kernel_sp) = user_invoke;
333 // clear out user regs that can be returned from the thread_ex_regs
334 // system call to prevent covert channel
335 Entry_frame *r = regs();
338 r->psr = Proc::Status_mode_user;
340 state_add(Thread_dead | Thread_suspended);
342 // ok, we're ready to go!
347 Thread::user_sp() const
348 { return regs()->sp(); }
352 Thread::user_sp(Mword sp)
353 { return regs()->sp(sp); }
355 IMPLEMENT inline NEEDS[Thread::exception_triggered]
357 Thread::user_ip() const
358 { return exception_triggered() ? _exc_cont.ip() : regs()->ip(); }
362 Thread::user_flags() const
365 IMPLEMENT inline NEEDS[Thread::exception_triggered]
367 Thread::user_ip(Mword ip)
369 if (exception_triggered())
373 Entry_frame *r = regs();
375 r->psr = (r->psr & ~Proc::Status_mode_mask) | Proc::Status_mode_user;
380 PUBLIC inline NEEDS ["trap_state.h"]
382 Thread::send_exception_arch(Trap_state *)
384 // nothing to tweak on ARM
388 PRIVATE static inline
390 Thread::save_fpu_state_to_utcb(Trap_state *ts, Utcb *u)
392 char *esu = (char *)&u->values[21];
393 Fpu::save_user_exception_state(ts, (Fpu::Exception_state_user *)esu);
398 Thread::invalid_ipc_buffer(void const *a)
401 return Mem_layout::in_kernel(((Address)a & Config::SUPERPAGE_MASK)
402 + Config::SUPERPAGE_SIZE - 1);
409 Thread::do_trigger_exception(Entry_frame *r, void *ret_handler)
411 if (!_exc_cont.valid())
413 _exc_cont.activate(r, ret_handler);
420 PRIVATE static inline
421 bool FIASCO_WARN_RESULT
422 Thread::copy_utcb_to_ts(L4_msg_tag const &tag, Thread *snd, Thread *rcv,
423 unsigned char rights)
425 Trap_state *ts = (Trap_state*)rcv->_utcb_handler;
426 Utcb *snd_utcb = snd->utcb().access();
427 Mword s = tag.words();
429 if (EXPECT_FALSE(rcv->exception_triggered()))
431 // triggered exception pending
432 Mem::memcpy_mwords (ts, snd_utcb->values, s > 15 ? 15 : s);
433 if (EXPECT_TRUE(s > 19))
435 // sanitize processor mode
437 snd_utcb->values[19] &= ~Proc::Status_mode_mask; // clear mode
438 snd_utcb->values[19] |= Proc::Status_mode_supervisor
439 | Proc::Status_interrupts_disabled;
441 Continuation::User_return_frame const *s
442 = reinterpret_cast<Continuation::User_return_frame const *>((char*)&snd_utcb->values[15]);
444 rcv->_exc_cont.set(ts, s);
449 Mem::memcpy_mwords (ts, snd_utcb->values, s > 18 ? 18 : s);
450 if (EXPECT_TRUE(s > 18))
451 ts->pc = snd_utcb->values[18];
452 if (EXPECT_TRUE(s > 19))
454 // sanitize processor mode
455 Mword p = snd_utcb->values[19];
456 p &= ~(Proc::Status_mode_mask | Proc::Status_interrupts_mask); // clear mode & irqs
457 p |= Proc::Status_mode_user;
462 if (tag.transfer_fpu() && (rights & L4_fpage::W))
463 snd->transfer_fpu(rcv);
465 if ((tag.flags() & 0x8000) && (rights & L4_fpage::W))
466 rcv->utcb().access()->user[2] = snd_utcb->values[25];
468 bool ret = transfer_msg_items(tag, snd, snd_utcb,
469 rcv, rcv->utcb().access(), rights);
471 rcv->state_del(Thread_in_exception);
476 PRIVATE static inline NEEDS[Thread::save_fpu_state_to_utcb]
477 bool FIASCO_WARN_RESULT
478 Thread::copy_ts_to_utcb(L4_msg_tag const &, Thread *snd, Thread *rcv,
479 unsigned char rights)
481 Trap_state *ts = (Trap_state*)snd->_utcb_handler;
484 Lock_guard <Cpu_lock> guard (&cpu_lock);
485 Utcb *rcv_utcb = rcv->utcb().access();
487 Mem::memcpy_mwords (rcv_utcb->values, ts, 15);
488 Continuation::User_return_frame *d
489 = reinterpret_cast<Continuation::User_return_frame *>((char*)&rcv_utcb->values[15]);
491 snd->_exc_cont.get(d, ts);
494 if (EXPECT_TRUE(!snd->exception_triggered()))
496 rcv_utcb->values[18] = ts->pc;
497 rcv_utcb->values[19] = ts->psr;
500 if (rcv_utcb->inherit_fpu() && (rights & L4_fpage::W))
501 snd->transfer_fpu(rcv);
503 save_fpu_state_to_utcb(ts, rcv_utcb);
510 Thread::invoke_arch(L4_msg_tag & /*tag*/, Utcb * /*utcb*/)
517 Thread::sys_control_arch(Utcb *)
522 // ------------------------------------------------------------------------
523 IMPLEMENTATION [arm && armv6plus]:
527 Thread::vcpu_resume_user_arch()
529 // just an experiment for now, we cannot really take the
530 // user-writable register because user-land might already use it
531 asm volatile("mcr p15, 0, %0, c13, c0, 2"
532 : : "r" (utcb().access(true)->values[25]) : "memory");
535 // ------------------------------------------------------------------------
536 IMPLEMENTATION [arm && !armv6plus]:
540 Thread::vcpu_resume_user_arch()
543 //-----------------------------------------------------------------------------
548 EXTENSION class Thread
551 static void kern_kdebug_ipi_entry() asm("kern_kdebug_ipi_entry");
554 PUBLIC static inline NEEDS["ipi.h"]
556 Thread::check_for_ipi(unsigned irq)
558 if (Ipi::is_ipi(irq))
563 Thread::handle_remote_requests_irq();
566 Ipi::eoi(Ipi::Debug);
567 kern_kdebug_ipi_entry();
569 case Ipi::Global_request:
570 handle_global_remote_requests_irq();
579 //-----------------------------------------------------------------------------
580 IMPLEMENTATION [!mp]:
584 Thread::check_for_ipi(unsigned)