*
* Instance manager implementation.
*
- * (c) 2011 Björn Döbel <doebel@os.inf.tu-dresden.de>,
+ * (c) 2011-2013 Björn Döbel <doebel@os.inf.tu-dresden.de>,
* economic rights: Technische Universität Dresden (Germany)
* This file is part of TUD:OS and distributed under the terms of the
* GNU General Public License 2.
#include "configuration"
#include <l4/sys/segment.h>
+#include <l4/re/mem_alloc>
+#include <l4/re/rm>
+#include <l4/re/env>
+#include <l4/re/dataspace>
+#include <l4/re/util/cap_alloc>
+#include <l4/plr/uu.h>
#define MSG() DEBUGf(Romain::Log::Manager)
+#include "fault_handlers/syscalls_factory.h"
Romain::Configuration Romain::globalconfig;
+
L4_INLINE unsigned countbits(long v)
{
v = v - ((v >> 1) & 0x55555555); // reuse input as temporary
return ((v + ((v >> 4) & 0xF0F0F0F)) * 0x1010101) >> 24; // count
}
+
L4_INLINE l4_umword_t count_online_cpus()
{
- l4_umword_t ret;
- l4_sched_cpu_set_t set = l4_sched_cpu_set(0, 0);
- if (l4_error(L4Re::Env::env()->scheduler()->info(&ret, &set)) < 0) {
+ l4_umword_t maxcpu = 0;
+ l4_sched_cpu_set_t cpuonline = l4_sched_cpu_set(0, 0);
+ if (l4_error(L4Re::Env::env()->scheduler()->info(&maxcpu, &cpuonline)) < 0) {
ERROR() << "reading CPU info";
}
- ret = countbits(set.map);
- INFO() << "Found " << ret << " CPUs.";
- return ret;
+ INFO() << "Online " << countbits(cpuonline.map) << " / MAX " << maxcpu;
+
+ return countbits(cpuonline.map) > maxcpu ? maxcpu : countbits(cpuonline.map);
}
+
Romain::InstanceManager::InstanceManager(unsigned int argc,
char const **argv,
unsigned num_instances)
- : _am(0),
+ : LogicalCPUMap(),
+ _am(0),
_instances(),
- _callback(0),
+ _threadgroups(),
_num_observers(0),
_num_inst(num_instances),
_num_cpu(1),
_argc(argc), // XXX: remove
- _argv(argv) // XXX: remove
+ _argv(argv), // XXX: remove
+ _logBuf(0)
{
configure();
* First, register those observers that don't interfere
* with anyone else and get notified all the time.
*/
+ DEBUG() << "[observer] vcpu state.";
BoolObserverConfig("general:print_vcpu_state",
this, "vcpu_state");
+ DEBUG() << "[observer] trap limit.";
ObserverConfig(this, "trap_limit");
/*
- * We always need the system call observer.
+ * Always needed -- slightly ordered by the number of
+ * calls they are expected to see, so that we minimize
+ * the amount of unnecessary observer callbacks.
*/
- ObserverConfig(this, "syscalls");
+ DEBUG() << "[observer] page faults.";
ObserverConfig(this, "pagefaults");
+ DEBUG() << "[observer] syscalls.";
+ ObserverConfig(this, "syscalls");
+ DEBUG() << "[observer] threads.";
+ BoolObserverConfig("general:threads", this, "threads");
+ DEBUG() << "[observer] trap.";
ObserverConfig(this, "trap");
+ DEBUG() << "[observer] simpledbg.";
StringObserverConfig("general:debug", this);
+ DEBUG() << "[observer] intercept-kip.";
BoolObserverConfig("general:intercept_kip", this, "kip-time");
+ DEBUG() << "[observer] swifi.";
BoolObserverConfig("general:swifi", this, "swifi");
+ DEBUG() << "[observer] logreplica.";
+ BoolObserverConfig("general:logreplica", this, "replicalog");
}
INFO() << "red: '" << redundancy << "'";
if (strcmp(redundancy, "none") == 0) {
_num_inst = 1;
- set_redundancy_callback(new NoRed());
} else if (strcmp(redundancy, "dual") == 0) {
_num_inst = 2;
- set_redundancy_callback(new DMR(2));
} else if (strcmp(redundancy, "triple") == 0) {
_num_inst = 3;
- set_redundancy_callback(new DMR(3));
} else {
ERROR() << "Invalid redundancy setting: " << redundancy;
enter_kdebug("Invalid redundancy setting");
}
}
+
+void Romain::InstanceManager::configure_logbuf(int sizeMB)
+{
+ INFO() << "Log buffer size: " << sizeMB << " MB requested.";
+ unsigned size_in_bytes = sizeMB << 20;
+
+ L4::Cap<L4Re::Dataspace> ds;
+
+ l4_addr_t addr = Romain::Region_map::allocate_and_attach(&ds, size_in_bytes, 0, L4_SUPERPAGESHIFT);
+ INFO() << "Log buffer attached to 0x" << std::hex << addr;
+
+ _logBuf->set_buffer(reinterpret_cast<unsigned char*>(addr), size_in_bytes);
+}
+
+
/*
* Romain ini file settings
* =====================
* - gdb -> GDB stub logging
* - all -> everything
*
+ * logbuf [int] (-1)
+ * - establish a log buffer with the given size in MB
+ * - runtime events are logged into this buffer and can later
+ * be dumped for postprocessing -> this is an alternative to
+ * printing a lot of stuff to the serial console
+ *
+ * logcpu [int]
+ * - event generation needs a global timestamp. On real SMP hardware
+ * CPUs disagree on their local TSC values. As a workaround, we start
+ * a dedicated thread that busily writes its local TSC to a global timer
+ * variable that is then read by everyone else. This of course requires
+ * the thread to solely run on a dedicated CPU. This option sets the
+ * respective CPU #.
+ *
+ * logrdtsc [bool] (false)
+ * - use local TSC instead of global time stamp counter for event timestamps
+ * -> use on Qemu where a dedicated timestamp thread does not work properly
+ *
+ * logreplica [bool] (false)
+ * - assign each replica a log buffer (mapped to REPLICA_LOG_ADDRESS)
+ *
+ * logtimeout [int] (-1)
+ * - run the replicated app for N seconds, then halt all threads and
+ * print replica stats (as if on termination)
+ *
+ * replicalogsize [int] (-1)
+ * - buffser size for the replica-specific log buffer
+ *
* swifi [bool] (false)
* - Perform fault injection experiments, details are configured
* in the [swifi] section.
*/
void Romain::InstanceManager::configure()
{
+#define USE_SHARABLE_TIMESTAMP 1
+
+ int logMB = ConfigIntValue("general:logbuf");
+
+#if USE_SHARABLE_TIMESTAMP
+ _logBuf = new Measurements::EventBuf(true);
+ L4::Cap<L4Re::Dataspace> tsds;
+ l4_addr_t ts_addr = Romain::Region_map::allocate_and_attach(&tsds, L4_PAGESIZE);
+ l4_touch_ro((void*)ts_addr, L4_PAGESIZE);
+ _logBuf->set_tsc_buffer(reinterpret_cast<l4_uint64_t*>(ts_addr));
+#else
+ _logBuf = new Measurements::EventBuf();
+#endif
+ if (logMB != -1) {
+ configure_logbuf(logMB);
+ }
+
+ Log::logLocalTSC = ConfigBoolValue("general:logrdtsc", false);
+
+ /*
+ * These modes are exclusive: either we use the local TSC _xor_ we start a
+ * timer thread on a dedicated CPU.
+ */
+ if (!Log::logLocalTSC) {
+ int logCPU = ConfigIntValue("general:logcpu");
+ if (logCPU != -1) {
+ INFO() << "Starting counter thread on CPU " << logCPU;
+ INFO() << "Timestamp @ 0x" << std::hex << (l4_addr_t)_logBuf->timestamp;
+ Measurements::EventBuf::launchTimerThread((l4_addr_t)_logBuf->timestamp,
+ logCPU);
+ }
+ }
+
char *log = strdup(ConfigStringValue("general:log", "none"));
configure_logflags(log);
}
+void Romain::InstanceManager::logdump()
+{
+ int logMB = ConfigIntValue("general:logbuf");
+ if (logMB != -1) {
+ char const *filename = "sampledump.txt";
+
+ unsigned oldest = _logBuf->oldest();
+ unsigned dump_start, dump_size;
+
+ if (oldest == 0) { // half-full -> dump from 0 to index
+ dump_start = 0;
+ dump_size = _logBuf->index * sizeof(Measurements::GenericEvent);
+ } else { // buffer completely full -> dump full size starting from oldest entry
+ dump_start = oldest * sizeof(Measurements::GenericEvent);
+ dump_size = _logBuf->size * sizeof(Measurements::GenericEvent);
+ }
+
+ uu_dumpz_ringbuffer(filename, _logBuf->buffer,
+ _logBuf->size * sizeof(Measurements::GenericEvent),
+ dump_start, dump_size);
+ }
+}
+
+
/*
* Prepare the stack that is used by the fault handler whenever a
* VCPU enters the master task.
*/
l4_addr_t Romain::InstanceManager::prepare_stack(l4_addr_t sp,
Romain::App_instance *inst,
- Romain::App_thread *thread)
+ Romain::App_thread *thread,
+ Romain::Thread_group *tgroup)
{
Romain::Stack st(sp);
st.push(_am);
+ st.push(tgroup);
st.push(thread);
st.push(inst);
st.push(this);
}
-void Romain::InstanceManager::run_instances()
+Romain::App_thread*
+Romain::InstanceManager::create_thread(l4_umword_t eip, l4_umword_t esp,
+ unsigned instance_id, Romain::Thread_group *group)
{
- for (unsigned i = 0; i < _num_inst; ++i) {
- Romain::App_thread *at = new Romain::App_thread(_init_eip, _init_esp,
+ Romain::App_thread *at = new Romain::App_thread(eip, esp,
reinterpret_cast<l4_addr_t>(VCPU_handler),
reinterpret_cast<l4_addr_t>(VCPU_startup)
);
- at->setup_gdt(_am->stack()->target_top() - 4, 4);
+ /*
+ * Set up the VCPU handler thread. It has been allocated in
+ * App_thread's constructor.
+ */
+ DEBUG() << "prepare: " << (void*)at->handler_sp();
+ at->handler_sp(prepare_stack(at->handler_sp(),
+ _instances[instance_id], at, group));
+
+ /*
+ * phys. CPU assignment, currently done by mapping instances to dedicated
+ * physical CPUs
+ */
+ if (_num_cpu > 1) {
+ INFO() << instance_id << " " << (instance_id+1) % _num_cpu << " " << _num_cpu;
+
+ unsigned logCPU = 1;
+
+ /* XXX REPLICAS PER CPU XXX */
+ //logCPU = logicalToCPU(group->uid % _num_cpu);
+
+ /* XXX INSTANCES PER CPU XXX */
+ //logCPU = logicalToCPU((instance_id + 1) % _num_cpu);
+
+ /* XXX OVERLAPPING REPLICAS XXX */
+ //logCPU = logicalToCPU((group->uid + instance_id) % _num_cpu);
+
+ /* XXX RANDOM PLACEMENT XXX */
+ //logCPU = logicalToCPU(random() % _num_cpu);
+
+ /* XXX Threads assigned RR to CPUs */
+ static int threadcount = 0;
+ logCPU = logicalToCPU(threadcount % _num_cpu);
+ threadcount++;
+
+ /* XXX The hard-coded placement map */
+ //int cpumap[12] = {0, 1, 2,
+ // 0, 0, 0,
+ // 3, 4, 5,
+ // 0, 0, 0};
+ //logCPU = logicalToCPU(cpumap[threadcount]);
+ //threadcount++;
+
+ at->cpu(logCPU);
+ } else {
+ at->cpu(0);
+ }
+
+ return at;
+}
+
+
+Romain::Thread_group *
+Romain::InstanceManager::create_thread_group(l4_umword_t eip, l4_umword_t esp, std::string n,
+ unsigned cap, unsigned uid)
+{
+ Romain::Thread_group *group = new Romain::Thread_group(n, cap, uid);
+ group->set_redundancy_callback(new DMR(_num_inst));
+
+ for (unsigned i = 0; i < _num_inst; ++i) {
+ /*
+ * Thread creation
+ */
+ Romain::App_thread *at = create_thread(eip, esp, i, group);
+ group->add_replica(at);
+ }
+
+ _threadgroups.push_back(group);
+ return group;
+}
+
+void Romain::InstanceManager::run_instances()
+{
+ Romain::Thread_group *group = create_thread_group(_init_eip, _init_esp, "init",
+ Romain::FIRST_REPLICA_CAP, 0);
+ DEBUG() << "created group object @ " << (void*)group;
+ theObjectFactory.register_thread_group(group, Romain::FIRST_REPLICA_CAP);
+
+ _check(group->threads.size() != _num_inst, "not enough threads created?");
+
+ for (unsigned i = 0; i < _num_inst; ++i) {
+
+ App_thread *at = group->threads[i];
+
+ /*
+ * Stack setup
+ */
+ at->thread_sp((l4_addr_t)_am->stack()->relocate(_am->stack()->ptr()));
+
+ /*
+ * The initial UTCB address is on top of the app's stack. This location
+ * is used for the first GDT entry, which L4Re later uses to find the
+ * thread's UTCB address.
+ */
+ at->setup_utcb_segdesc(_am->stack()->target_top() - 4, 4);
+
+ /*
+ * Establish UTCB mapping
+ */
Romain::Region_handler &rh = const_cast<Romain::Region_handler&>(
_am->rm()->find(_am->prog_info()->utcbs_start)->second);
_check(_am->rm()->copy_existing_mapping(rh, 0, i) != true,
"could not create UTCB copy");
at->remote_utcb(rh.local_region(i).start());
- DEBUG() << "prepare: " << (void*)at->handler_sp();
- at->handler_sp(prepare_stack(at->handler_sp(), _instances[i], at));
- at->thread_sp((l4_addr_t)_am->stack()->relocate(_am->stack()->ptr()));
- if (_num_cpu > 1) {
- INFO() << i << " " << (i+1) % _num_cpu << " " << _num_cpu;
- at->cpu((i+1) % _num_cpu);
- } else {
- at->cpu(0);
- }
-
- startup_notify(_instances[i], at, _am);
+ /*
+ * Notfiy handlers about an instance that has started
+ */
+ startup_notify(_instances[i], at, group, _am);
+ /*
+ * Start the thread itself
+ */
+ at->vcpu()->r()->sp = at->thread_sp();
at->start();
+ at->commit_client_gdt();
}
+
+ group->activate();
}
projects / l4.git / blobdiff