l4/pkg/plr/server/src/manager.cc

   1 /*
   2  * manager.cc --
   3  *
   4  *     Instance manager implementation.
   5  *
   6  * (c) 2011-2013 Björn Döbel <doebel@os.inf.tu-dresden.de>,
   7  *     economic rights: Technische Universität Dresden (Germany)
   8  * This file is part of TUD:OS and distributed under the terms of the
   9  * GNU General Public License 2.
  10  * Please see the COPYING-GPL-2 file for details.
  11  */
  12
  13 #include "manager"
  14 #include "app_loading"
  15 #include "configuration"
  16
  17 #include <l4/sys/segment.h>
  18 #include <l4/re/mem_alloc>
  19 #include <l4/re/rm>
  20 #include <l4/re/env>
  21 #include <l4/re/dataspace>
  22 #include <l4/re/util/cap_alloc>
  23 #include <l4/plr/uu.h>
  24
  25 #define MSG() DEBUGf(Romain::Log::Manager)
  26 #include "fault_handlers/syscalls_factory.h"
  27
  28 Romain::Configuration Romain::globalconfig;
  29
  30
  31 L4_INLINE unsigned countbits(long v)
  32 {
  33         v = v - ((v >> 1) & 0x55555555);                         // reuse input as temporary
  34         v = (v & 0x33333333) + ((v >> 2) & 0x33333333);          // temp
  35         return ((v + ((v >> 4) & 0xF0F0F0F)) * 0x1010101) >> 24; // count
  36 }
  37
  38
  39 L4_INLINE l4_umword_t count_online_cpus()
  40 {
  41         l4_umword_t maxcpu = 0;
  42         l4_sched_cpu_set_t cpuonline = l4_sched_cpu_set(0, 0);
  43         if (l4_error(L4Re::Env::env()->scheduler()->info(&maxcpu, &cpuonline)) < 0) {
  44                 ERROR() << "reading CPU info";
  45         }
  46
  47         INFO() << "Online " << countbits(cpuonline.map) << " / MAX " << maxcpu;
  48
  49         return countbits(cpuonline.map) > maxcpu ? maxcpu : countbits(cpuonline.map);
  50 }
  51
  52
  53 Romain::InstanceManager::InstanceManager(unsigned int argc,
  54                                          char const **argv,
  55                                          unsigned num_instances)
  56         : _am(0),
  57           _instances(),
  58           _num_observers(0),
  59           _num_inst(num_instances),
  60           _num_cpu(1),
  61           _argc(argc), // XXX: remove
  62           _argv(argv), // XXX: remove
  63           _logBuf(0)
  64 {
  65         configure();
  66
  67         _gdt_min = fiasco_gdt_get_entry_offset(L4_INVALID_CAP, l4_utcb());
  68         MSG() << "GDT MIN: " << _gdt_min;
  69
  70         _num_cpu = count_online_cpus();
  71         /*
  72          * initial parameter is argv for the client program, this means
  73          * *argv is the file name to load.
  74          */
  75         _name = *argv;
  76
  77         _am = new Romain::App_model(_name, argc, argv);
  78         Romain::Elf_Ldr loader(_am);
  79         //loader.load();
  80         loader.launch();
  81
  82         _init_eip = _am->prog_info()->entry;
  83         _init_esp = _am->prog_info()->stack_addr;
  84         INFO() << "Program entry point at 0x" << std::hex << _init_eip;
  85         INFO() << "              stack at 0x" << std::hex << _init_esp;
  86
  87 #if SPLIT_HANDLING
  88         int res = pthread_create(&_split_handler, 0, split_handler_fn, this);
  89         _check(res != 0, "could not create split handler thread");
  90 #endif
  91 }
  92
  93
  94 void Romain::InstanceManager::configure_logflags(char *flags)
  95 {
  96         printf("flags %p\n", flags);
  97         if (!flags) {
  98                 Romain::Log::logFlags = 0;
  99         } else {
 100                 unsigned max = strlen(flags);
 101                 for (unsigned j = 0; j < max; ++j) {
 102                         if (flags[j] == ',') flags[j] = 0;
 103                 }
 104
 105                 char const *c = flags;
 106                 while (c <= flags + max) {
 107                         printf("  %s\n", c);
 108                         if ((strcmp(c, "mem") == 0) || (strcmp(c, "memory") == 0)) {
 109                                 Romain::Log::logFlags |= Romain::Log::Memory;
 110                         } else if (strcmp(c, "emulator") == 0) {
 111                                 Romain::Log::logFlags |= Romain::Log::Emulator;
 112                         } else if (strcmp(c, "manager") == 0) {
 113                                 Romain::Log::logFlags |= Romain::Log::Manager;
 114                         } else if (strcmp(c, "faults") == 0) {
 115                                 Romain::Log::logFlags |= Romain::Log::Faults;
 116                         } else if (strcmp(c, "redundancy") == 0) {
 117                                 Romain::Log::logFlags |= Romain::Log::Redundancy;
 118                         } else if (strcmp(c, "loader") == 0) {
 119                                 Romain::Log::logFlags |= Romain::Log::Loader;
 120                         } else if (strcmp(c, "swifi") == 0) {
 121                                 Romain::Log::logFlags |= Romain::Log::Swifi;
 122                         } else if (strcmp(c, "gdb") == 0) {
 123                                 Romain::Log::logFlags |= Romain::Log::Gdb;
 124                         } else if (strcmp(c, "all") == 0) {
 125                                 Romain::Log::logFlags = Romain::Log::All;
 126                         }
 127
 128                         c += (strlen(c)+1);
 129                 }
 130                 printf("Flags: %08lx\n", Romain::Log::logFlags);
 131         }
 132 }
 133
 134
 135 void Romain::InstanceManager::configure_fault_observers()
 136 {
 137         /*
 138          * First, register those observers that don't interfere
 139          * with anyone else and get notified all the time.
 140          */
 141         BoolObserverConfig("general:print_vcpu_state",
 142                            this, "vcpu_state");
 143         ObserverConfig(this, "trap_limit");
 144
 145         /*
 146          * Always needed -- slightly ordered by the number of
 147          * calls they are expected to see, so that we minimize
 148          * the amount of unnecessary observer callbacks.
 149          */
 150         ObserverConfig(this, "pagefaults");
 151         ObserverConfig(this, "syscalls");
 152         BoolObserverConfig("general:threads", this, "threads");
 153         ObserverConfig(this, "trap");
 154
 155         StringObserverConfig("general:debug", this);
 156         BoolObserverConfig("general:intercept_kip", this, "kip-time");
 157         BoolObserverConfig("general:swifi", this, "swifi");
 158         BoolObserverConfig("general:logreplica", this, "replicalog");
 159 }
 160
 161
 162 void Romain::InstanceManager::configure_redundancy()
 163 {
 164         char const *redundancy = ConfigStringValue("general:redundancy");
 165         if (!redundancy) redundancy = "none";
 166         INFO() << "red: '" << redundancy << "'";
 167         if (strcmp(redundancy, "none") == 0) {
 168                 _num_inst = 1;
 169         } else if (strcmp(redundancy, "dual") == 0) {
 170                 _num_inst = 2;
 171         } else if (strcmp(redundancy, "triple") == 0) {
 172                 _num_inst = 3;
 173         } else {
 174                 ERROR() << "Invalid redundancy setting: " << redundancy;
 175                 enter_kdebug("Invalid redundancy setting");
 176         }
 177 }
 178
 179
 180 void Romain::InstanceManager::configure_logbuf(int sizeMB)
 181 {
 182         INFO() << "Log buffer size: " << sizeMB << " MB requested.";
 183         unsigned size_in_bytes = sizeMB << 20;
 184
 185         L4::Cap<L4Re::Dataspace> ds;
 186
 187         l4_addr_t addr = Romain::Region_map::allocate_and_attach(&ds, size_in_bytes, 0, L4_SUPERPAGESHIFT);
 188     INFO() << "Log buffer attached to 0x" << std::hex << addr;
 189
 190     _logBuf->set_buffer(reinterpret_cast<unsigned char*>(addr), size_in_bytes);
 191 }
 192
 193
 194 /*
 195  * Romain ini file settings
 196  * =====================
 197  *
 198  *  'general' section
 199  *  -----------------
 200  *
 201  *  The 'general' section determines which fault handlers are registered.
 202  *
 203  *  print_vcpu_state [bool]
 204  *     - Registers a handler printing the state of a VCPU upon every
 205  *       fault entry
 206  *
 207  *  debug [string = {simple,gdb}]
 208  *     - Configures a debugger stub. 'simple' refers to builtin debugging,
 209  *       'gdb' starts a gdb stub. Further configuration for the debuggers
 210  *       is done in separate INI sections.
 211  *
 212  *  page_fault_handling [string = {ro}]
 213  *     - Specify the way in which paging is done.
 214  *       'ro' means that client memory is mapped read-only and write
 215  *       accesses to the respective regions are emulated.
 216  *
 217  *  redundancy [string = {dual, triple}]
 218  *     - configure the number of replicas that are started
 219  *
 220  *
 221  *  log [string list]
 222  *     - comma-separated list of strings for configuring logging
 223  *     - available flags are:
 224  *       - mem|memory -> memory management
 225  *       - emulator   -> instruction emulation
 226  *       - manager    -> replica management
 227  *       - faults     -> generic fault entry path
 228  *       - redundancy -> DMR/TMR-specific logs
 229  *       - swifi      -> fault injetion
 230  *       - gdb        -> GDB stub logging
 231  *       - all        -> everything
 232  *
 233  *  logbuf [int] (-1)
 234  *       - establish a log buffer with the given size in MB
 235  *       - runtime events are logged into this buffer and can later
 236  *         be dumped for postprocessing -> this is an alternative to
 237  *         printing a lot of stuff to the serial console
 238  *
 239  *  logcpu [int]
 240  *       - event generation needs a global timestamp. On real SMP hardware
 241  *         CPUs disagree on their local TSC values. As a workaround, we start
 242  *         a dedicated thread that busily writes its local TSC to a global timer
 243  *         variable that is then read by everyone else. This of course requires
 244  *         the thread to solely run on a dedicated CPU. This option sets the
 245  *         respective CPU #.
 246  *
 247  *  logrdtsc [bool] (false)
 248  *       - use local TSC instead of global time stamp counter for event timestamps
 249  *         -> use on Qemu where a dedicated timestamp thread does not work properly
 250  *
 251  *  logreplica [bool] (false)
 252  *       - assign each replica a log buffer (mapped to REPLICA_LOG_ADDRESS)
 253  *
 254  *  replicalogsize [int] (-1)
 255  *       - buffser size for the replica-specific log buffer
 256  *
 257  *  swifi [bool] (false)
 258  *       - Perform fault injection experiments, details are configured
 259  *         in the [swifi] section.
 260  *
 261  *  kip-time [bool] (false)
 262  *       - Turn on/off KIP timer access. This is used to turn replica
 263  *         accesses to the clock field of the KIP into traps (by placing
 264  *         software breakpoints on specifically configured instructions).
 265  *         Use this, if your application needs clock info from the KIP.
 266  *
 267  *  max_traps [int] (-1)
 268  *       - Handle a maximum amount of traps before terminating the
 269  *         replicated application. Use as a debugging aid.
 270  *
 271  *  print_time [bool] (true)
 272  *       - include timing information in printed output.
 273  *
 274  *  'gdbstub' section
 275  *  -----------------
 276  *
 277  *  This section configures the behavior of the GDB stub.
 278  *
 279  *  port [int]
 280  *     - Configures the GDB stub to use a TCP/IP connection and wait
 281  *       for a remote GDB to connect on the port specified. If this
 282  *       option is _not_ set, the GDB stub will try to use a serial
 283  *       connection through COM2.
 284  *
 285  *  XXX make COM port configurable
 286  *
 287  *  'simpledbg' section
 288  *  -------------------
 289  *
 290  *  This section configures Romain's builtin debugger, which is programmed through
 291  *  INI file commands only and performs a narrow range of debugging tasks only.
 292  *
 293  *  singlestep [int]
 294  *     - Patches an INT3 breakpoint to the given address. Then executes the program
 295  *       until the breakpoint is hit and thereafter switches to single-stepping
 296  *       mode.
 297  *
 298  *  'kip-time' section
 299  *  ------------------
 300  *
 301  *  The KIP-time instrumentation needs a list of addresses that point to
 302  *  KIP->clock accessing instructions. These are supplied as a comma-separated
 303  *  list of hex values for the target command.
 304  *
 305  *  target [comma-separated list of hex addresses]
 306  *
 307  *  'swifi' section
 308  *  ---------------
 309  *
 310  *  Configures fault-injection experiments that are performed on replicas.
 311  *  By default, SWIFI currently injects faults into replica #0.
 312  *
 313  *  - target [hex]
 314  *    specifies an address to place a breakpoint on. Upon hitting this
 315  *    BP, a SWIFI injection is performed.
 316  *
 317  *  - inject [string]
 318  *    specifies what kind of injection to perform when hitting the BP.
 319  *    Available values:
 320  *      - 'gpr' -> flip a random bit in a randomly selected
 321  *                 general-purpose register
 322  */
 323 void Romain::InstanceManager::configure()
 324 {
 325 #define USE_SHARABLE_TIMESTAMP 1
 326
 327         int logMB = ConfigIntValue("general:logbuf");
 328
 329 #if USE_SHARABLE_TIMESTAMP
 330         _logBuf = new Measurements::EventBuf(true);
 331         L4::Cap<L4Re::Dataspace> tsds;
 332         l4_addr_t ts_addr = Romain::Region_map::allocate_and_attach(&tsds, L4_PAGESIZE);
 333         l4_touch_ro((void*)ts_addr, L4_PAGESIZE);
 334         _logBuf->set_tsc_buffer(reinterpret_cast<l4_uint64_t*>(ts_addr));
 335 #else
 336         _logBuf = new Measurements::EventBuf();
 337 #endif
 338         if (logMB != -1) {
 339                 configure_logbuf(logMB);
 340         }
 341
 342         Log::logLocalTSC = ConfigBoolValue("general:logrdtsc", false);
 343
 344         /*
 345          * These modes are exclusive: either we use the local TSC _xor_ we start a
 346          * timer thread on a dedicated CPU.
 347          */
 348         if (!Log::logLocalTSC) {
 349                 int logCPU = ConfigIntValue("general:logcpu");
 350                 if (logCPU != -1) {
 351                         INFO() << "Starting counter thread on CPU " << logCPU;
 352                         INFO() << "Timestamp @ 0x" << std::hex << (l4_addr_t)_logBuf->timestamp;
 353                         Measurements::EventBuf::launchTimerThread((l4_addr_t)_logBuf->timestamp,
 354                                                                   logCPU);
 355                 }
 356         }
 357
 358         char *log = strdup(ConfigStringValue("general:log", "none"));
 359         configure_logflags(log);
 360
 361         Log::withtime = ConfigBoolValue("general:print_time", true);
 362
 363         configure_fault_observers();
 364         configure_redundancy();
 365         free(log);
 366 }
 367
 368
 369 void Romain::InstanceManager::logdump()
 370 {
 371         int logMB = ConfigIntValue("general:logbuf");
 372         if (logMB != -1) {
 373                 char const *filename = "sampledump.txt";
 374
 375                 unsigned oldest = _logBuf->oldest();
 376                 unsigned dump_start, dump_size;
 377
 378                 if (oldest == 0) { // half-full -> dump from 0 to index
 379                         dump_start = 0;
 380                         dump_size  = _logBuf->index * sizeof(Measurements::GenericEvent);
 381                 } else { // buffer completely full -> dump full size starting from oldest entry
 382                         dump_start = oldest * sizeof(Measurements::GenericEvent);
 383                         dump_size  = _logBuf->size * sizeof(Measurements::GenericEvent);
 384                 }
 385
 386                 uu_dumpz_ringbuffer(filename, _logBuf->buffer,
 387                                     _logBuf->size * sizeof(Measurements::GenericEvent),
 388                                     dump_start, dump_size);
 389         }
 390 }
 391
 392
 393 /*
 394  * Prepare the stack that is used by the fault handler whenever a
 395  * VCPU enters the master task.
 396  *
 397  * This pushes relevant pointers to the stack so that the handler
 398  * functions can use them as parameters.
 399  */
 400 l4_addr_t Romain::InstanceManager::prepare_stack(l4_addr_t sp,
 401                                                  Romain::App_instance *inst,
 402                                                  Romain::App_thread *thread,
 403                                                  Romain::Thread_group *tgroup)
 404 {
 405         Romain::Stack st(sp);
 406
 407         st.push(_am);
 408         st.push(tgroup);
 409         st.push(thread);
 410         st.push(inst);
 411         st.push(this);
 412         st.push(0); // this would be the return address, but
 413                     // handlers return by vcpu_resume()
 414
 415         return st.sp();
 416 }
 417
 418
 419 void Romain::InstanceManager::create_instances()
 420 {
 421         for (unsigned i = 0; i < _num_inst; ++i) {
 422                 _instances.push_back(new Romain::App_instance(_name, i));
 423         }
 424 }
 425
 426
 427 Romain::App_thread*
 428 Romain::InstanceManager::create_thread(l4_umword_t eip, l4_umword_t esp,
 429                                        unsigned instance_id, Romain::Thread_group *group)
 430 {
 431                 Romain::App_thread *at = new Romain::App_thread(eip, esp,
 432                                                           reinterpret_cast<l4_addr_t>(VCPU_handler),
 433                                                           reinterpret_cast<l4_addr_t>(VCPU_startup)
 434                 );
 435
 436                 /*
 437                  * Set up the VCPU handler thread. It has been allocated in
 438                  * App_thread's constructor.
 439                  */
 440                 DEBUG() << "prepare: " << (void*)at->handler_sp();
 441                 at->handler_sp(prepare_stack(at->handler_sp(),
 442                                              _instances[instance_id], at, group));
 443
 444                 /*
 445                  * phys. CPU assignment, currently done by mapping instances to dedicated
 446                  * physical CPUs
 447                  */
 448                 if (_num_cpu > 1) {
 449                         INFO() << instance_id << " " << (instance_id+1) % _num_cpu << " " << _num_cpu;
 450
 451                         /* XXX REPLICAS PER CPU XXX */
 452                         at->cpu(group->uid % _num_cpu);
 453
 454                         /* XXX INSTANCES PER CPU XXX */
 455                         //at->cpu((instance_id + 1) % _num_cpu);
 456
 457                         /* XXX OVERLAPPING REPLICAS XXX */
 458                         //at->cpu((group->uid + instance_id) % _num_cpu);
 459
 460                         /* XXX RANDOM PLACEMENT XXX */
 461                         //at->cpu(random() % _num_cpu);
 462
 463                         /* XXX Threads assigned RR to CPUs */
 464                         //static int threadcount = 1;
 465                         //at->cpu(threadcount % _num_cpu);
 466                         //threadcount++;
 467                 } else {
 468                         at->cpu(0);
 469                 }
 470
 471                 return at;
 472 }
 473
 474
 475 Romain::Thread_group *
 476 Romain::InstanceManager::create_thread_group(l4_umword_t eip, l4_umword_t esp, std::string n,
 477                                              unsigned cap, unsigned uid)
 478 {
 479         Romain::Thread_group *group = new Romain::Thread_group(n, cap, uid);
 480         group->set_redundancy_callback(new DMR(_num_inst));
 481
 482         for (unsigned i = 0; i < _num_inst; ++i) {
 483                 /*
 484                  * Thread creation
 485                  */
 486                 Romain::App_thread *at = create_thread(eip, esp, i, group);
 487                 group->add_replica(at);
 488         }
 489
 490         return group;
 491 }
 492
 493
 494 void Romain::InstanceManager::run_instances()
 495 {
 496         Romain::Thread_group *group = create_thread_group(_init_eip, _init_esp, "init",
 497                                                                                                           Romain::FIRST_REPLICA_CAP, 0);
 498         DEBUG() << "created group object @ " << (void*)group;
 499         theObjectFactory.register_thread_group(group, Romain::FIRST_REPLICA_CAP);
 500
 501         _check(group->threads.size() != _num_inst, "not enough threads created?");
 502
 503         for (unsigned i = 0; i < _num_inst; ++i) {
 504
 505                 App_thread *at = group->threads[i];
 506
 507                 /*
 508                  * Stack setup
 509                  */
 510                 at->thread_sp((l4_addr_t)_am->stack()->relocate(_am->stack()->ptr()));
 511
 512                 /*
 513                  * The initial UTCB address is on top of the app's stack. This location
 514                  * is used for the first GDT entry, which L4Re later uses to find the
 515                  * thread's UTCB address.
 516                  */
 517                 at->setup_utcb_segdesc(_am->stack()->target_top() - 4, 4);
 518
 519                 /*
 520                  * Establish UTCB mapping
 521                  */
 522                 Romain::Region_handler &rh = const_cast<Romain::Region_handler&>(
 523                                                   _am->rm()->find(_am->prog_info()->utcbs_start)->second);
 524                 _check(_am->rm()->copy_existing_mapping(rh, 0, i) != true,
 525                        "could not create UTCB copy");
 526                 at->remote_utcb(rh.local_region(i).start());
 527
 528                 /*
 529                  * Notfiy handlers about an instance that has started
 530                  */
 531                 startup_notify(_instances[i], at, group, _am);
 532
 533                 /*
 534                  * Start the thread itself
 535                  */
 536                 at->vcpu()->r()->sp = at->thread_sp();
 537                 at->start();
 538                 at->commit_client_gdt();
 539         }
 540
 541         group->activate();
 542 }