l4/pkg/plr/server/src/manager

   1 // vim: ft=cpp
   2
   3 /*
   4  * manager --
   5  *
   6  *     Definition of the instance manager that knows about all
   7  *     redundant VCPUs as well as the fault observers.
   8  *
   9  * (c) 2011-2013 Björn Döbel <doebel@os.inf.tu-dresden.de>,
  10  *     economic rights: Technische Universität Dresden (Germany)
  11  * This file is part of TUD:OS and distributed under the terms of the
  12  * GNU General Public License 2.
  13  * Please see the COPYING-GPL-2 file for details.
  14  */
  15
  16 /*
  17  * Romain instance manager
  18  */
  19
  20 #pragma once
  21
  22 #include "constants.h"
  23 #include "app"
  24 #include "thread_group.h"
  25 //#include "app_loading"
  26 #include "fault_observers"
  27 #include "cpuid.h"
  28 #include "watchdog.h"
  29
  30 #include <vector>
  31 #include <cstdio>
  32 #include <algorithm>
  33
  34 #include "redundancy.h"
  35 #include <l4/plr/measurements.h>
  36
  37 #if SPLIT_HANDLING
  38 EXTERN_C void *split_handler_fn(void*);
  39 #endif // SPLIT_HANDLING
  40
  41 namespace Romain
  42 {
  43         /*
  44          * Map logical (=sequential) CPU numbers to the topology of the underlying
  45          * platform.
  46          *
  47          * Background: Fiasco.OC uses sequential CPU numbers to assign threads to CPUs. These
  48          *             numbers are assigned during bootup and their order depends on the order
  49          *             in which booted CPUs come up. This means, assigning two threads to CPUs
  50          *             0 and 1 can in one case mean that they are running on the same core in
  51          *             different hyperthreads, while in the next run, they run on the different
  52          *             physical CPUs (or even NUMA domains, if we had those).
  53          *
  54          *             The purpose of this map is to make the assignment less surprising, e.g.,
  55          *             for the same platform, assigning to two specific CPU IDs should always
  56          *             lead to the same performance.
  57          */
  58         class LogicalCPUMap
  59         {
  60                 protected:
  61                 std::vector<CPU_id> _fiasco_cpus;
  62
  63                 static l4_umword_t fiasco_count_online_cpus()
  64                 {
  65                         l4_umword_t ret = 0, max = 0;
  66                         l4_sched_cpu_set_t cpus;
  67                         chksys(L4Re::Env::env()->scheduler()->info(&max, &cpus), "CPU count failed");
  68
  69                         if (max > 31) {
  70                                 enter_kdebug("MAX > 31");
  71                         }
  72                         for (unsigned i = 0; i < max; ++i) {
  73                                 if (cpus.map & (1 << i))
  74                                         ret += (1 << cpus.granularity());
  75                         }
  76                         return ret;
  77                 }
  78
  79                 static void* topology_scanner(void* data)
  80                 {
  81                         l4_debugger_set_object_name(pthread_l4_cap(pthread_self()), "romain::topscan");
  82
  83                         for (l4_umword_t cpu = 0; cpu < fiasco_count_online_cpus(); ++cpu) {
  84                                 Romain::LogicalCPUMap *m = reinterpret_cast<Romain::LogicalCPUMap*>(data);
  85
  86                                 //INFO() << "CPU " << cpu;
  87                                 L4::Cap<L4::Thread> thread = Pthread::L4::cap(pthread_self());
  88                                 l4_sched_param_t sp = l4_sched_param(2);
  89                                 sp.affinity = l4_sched_cpu_set(cpu, 0);
  90                                 chksys(L4Re::Env::env()->scheduler()->run_thread(thread, sp));
  91
  92                                 m->_fiasco_cpus[cpu] = CPUID::current_apicid(cpu);
  93                         }
  94                         return 0;
  95                 }
  96
  97
  98                 void scan_topology()
  99                 {
 100                         pthread_t p;
 101                         l4_mword_t ret = pthread_create(&p, 0, Romain::LogicalCPUMap::topology_scanner, this);
 102                         _check(ret != 0, "error creating topology scanner");
 103                         ret = pthread_join(p, 0);
 104
 105                         for (l4_umword_t i = fiasco_count_online_cpus(); i < MAX_CPUS; ++i) {
 106                                 _fiasco_cpus[i].apic_id = 0xDEADBEEF;
 107                         }
 108
 109                         INFO() << "Topology scan completed.";
 110                 }
 111
 112
 113                 void print_map()
 114                 {
 115                         INFO() << "Virtual to Fiasco CPU map";
 116
 117                         for (l4_umword_t i = 0; i < fiasco_count_online_cpus(); ++i) {
 118                                 INFO() << "virt# " << i << " fiasco# " << _fiasco_cpus[i].fiasco_id
 119                                        << " apic# " << std::hex << _fiasco_cpus[i].apic_id;
 120                         }
 121                 }
 122
 123                 public:
 124                         LogicalCPUMap()
 125                                 : _fiasco_cpus(MAX_CPUS)
 126                         {
 127                                 /* Default: identity mapping if nothing else works */
 128                                 for (l4_umword_t i = 0; i < MAX_CPUS; ++i) {
 129                                         _fiasco_cpus[i].fiasco_id = i;
 130                                         _fiasco_cpus[i].apic_id   = i;
 131                                 }
 132
 133                                 if (CPUID::have_cpuid() and
 134                                     (CPUID::max_cpuid() >= CPUID::TOPOLOGY)) {
 135
 136                                         scan_topology();
 137                                         auto f_it = _fiasco_cpus.begin();
 138                                         while ((*f_it).apic_id != 0xdeadbeef)
 139                                                 ++f_it;
 140                                         std::sort(_fiasco_cpus.begin(), f_it, CPU_id::comp_apic);
 141                                         // If Fiasco-CPU ID 0 is in the second half, reverse the order to
 142                                         // have the CPU0-Socket first
 143                                         // XXX Hack! This requires real socket knowledge.
 144                                         l4_umword_t logical0;
 145                                         for (logical0 = 0; logical0 < MAX_CPUS; ++logical0) {
 146                                                 if (_fiasco_cpus[logical0].fiasco_id == 0)
 147                                                         break;
 148                                         }
 149                                         if (logical0 >= CPUID::num_cpus() / 2) {
 150                                                 std::reverse(_fiasco_cpus.begin(), f_it);
 151                                         }
 152                                         print_map();
 153                                 }
 154
 155                                 //enter_kdebug("cpu");
 156
 157                         }
 158
 159                         l4_umword_t logicalToCPU(l4_umword_t log)
 160                         {
 161                                 INFO() << log << " -> " << _fiasco_cpus[log].fiasco_id;
 162                                 return _fiasco_cpus[log].fiasco_id;
 163                         }
 164         };
 165
 166         /*
 167          * Set up and keep track of the running instances
 168          */
 169         class InstanceManager : public LogicalCPUMap
 170         {
 171                 private:
 172                         Romain::App_model *_am;
 173                         std::vector<Romain::App_instance *> _instances;    // instance list
 174                         std::vector<Romain::Thread_group *> _threadgroups; // list of thread groups
 175
 176                         Romain::Observer *_observer_list[Romain::MAX_OBSERVERS]; // exception observer list
 177                         //Romain::RedundancyCallback *_callback;                 // the callback for redundancy checking
 178                         l4_umword_t _num_observers;
 179
 180                         char const *_name;        // app name
 181                         l4_umword_t    _num_inst;    // number of instances to run
 182                         l4_umword_t _num_cpu;     // number of available CPUs
 183
 184                         l4_addr_t _init_eip;      // initial EIP (from ELF)
 185                         l4_addr_t _init_esp;      // initial ESP (from ELF)
 186                         l4_umword_t _gdt_min;        // start of GDT entries
 187
 188                         l4_umword_t     _argc;       // client argc
 189                         char const **_argv;       // argv of the client
 190
 191 #if SPLIT_HANDLING
 192                         pthread_t _split_handler; // resilient core handler thread
 193 #endif // SPLIT_HANDLING
 194
 195 #if WATCHDOG
 196                         bool  _watchdog_enable;
 197                         int   _watchdog_timeout;
 198
 199                         Romain::Watchdog::SynchronizationMode _watchdog_mode;
 200
 201                         Romain::Watchdog* _watchdog;
 202 #endif
 203
 204                         /*
 205                          * Prepare handler stack by pushing 3 values:
 206                          *
 207                          * 1st parameter is an app_thread
 208                          * 2nd parameter is an app_instance
 209                          * 3rd parameter is a dummy for the location where usually the
 210                          *                  return address would be located
 211                          * 4th parameter is the thread group we are dealing with
 212                          */
 213                         l4_addr_t prepare_stack(l4_addr_t sp, Romain::App_instance *inst,
 214                                                 Romain::App_thread *thread, Romain::Thread_group* tgroup);
 215
 216
 217                         void configure_logflags(char *flags);
 218                         void configure_fault_observers();
 219                         void configure_redundancy();
 220                         void configure_logbuf(l4_mword_t size);
 221                         void configure_watchdog();
 222
 223                 public:
 224                         InstanceManager(l4_umword_t argc, char const **argv, l4_umword_t num_instances = 1);
 225
 226
 227                         /*
 228                          * Prepare the replica instances (VCPUs)
 229                          */
 230                         void create_instances();
 231
 232                         /*
 233                          * Create a single replica thread along with
 234                          * its VCPU handler stack
 235                          */
 236                         Romain::App_thread* create_thread(l4_umword_t eip, l4_umword_t esp,
 237                                                                                           l4_umword_t inst, Romain::Thread_group *group);
 238
 239                         /*
 240                          * Create a group of replica threads, e.g., all replicas
 241                          * of a single original thread
 242                          */
 243                         Romain::Thread_group* create_thread_group(l4_umword_t eip, l4_umword_t esp,
 244                                                                                                           std::string, l4_umword_t cap, l4_umword_t uid);
 245
 246                         /*
 247                          * Setup the VCPU handler stack.
 248                          *
 249                          * Puts fake parameters on the handler stack so that
 250                          * upon an event it looks like we got them passed as
 251                          * parameters.
 252                          */
 253                         void prepare_handler_stack(Romain::App_thread *t, Romain::App_instance* i);
 254
 255                         /*
 256                          * Start the instances
 257                          */
 258                         void run_instances();
 259
 260
 261                         void register_fault_observer(Romain::Observer* ob)
 262                         {
 263                                 _check(_num_observers >= MAX_OBSERVERS, "no more observers");
 264                                 _observer_list[_num_observers++] = ob;
 265                         }
 266
 267 #if WATCHDOG
 268                         Romain::Watchdog* watchdog() { return _watchdog; }
 269 #endif
 270 #if 0
 271                         Romain::RedundancyCallback *redundancy() const { return _callback; }
 272
 273
 274                         void set_redundancy_callback(Romain::RedundancyCallback* cb)
 275                         {
 276                                 _check(_callback != 0, "there is already a callback registered?");
 277                                 _callback = cb;
 278                         }
 279 #endif
 280
 281
 282                         void startup_notify(Romain::App_instance *i, Romain::App_thread *t,
 283                                             Romain::Thread_group* tg, Romain::App_model *a) const
 284                         {
 285                                 // XXX would need to be aware of observer list modifications
 286                                 //     -> for now, we assume that observers are not added/removed
 287                                 //        at runtime. otherwise we'd use sth. like an r/w lock with
 288                                 //        reader prioritization??
 289                                 for (l4_umword_t idx = 0; idx < _num_observers; ++idx) {
 290                                         DEBUG() << "[" << i->id() << "] startup notifying: " << _observer_list[idx]->name();
 291                                         _observer_list[idx]->startup_notify(i,t,tg,a);
 292                                 }
 293                         }
 294
 295
 296                         /*
 297                          * Entry point for CPU exception handling.
 298                          *
 299                          * Iterates over the list of registered observers and calls
 300                          * every observer's notify() function. Observers may stop
 301                          * this iteration process. See Romain::Observer::ObserverReturnVal
 302                          */
 303                         Romain::Observer::ObserverReturnVal
 304                         fault_notify(Romain::App_instance *i, Romain::App_thread *t,
 305                                      Romain::Thread_group* tg, Romain::App_model *a)
 306                         {
 307                                 Romain::Observer::ObserverReturnVal v = Romain::Observer::Ignored;
 308                                 // XXX would need to be aware of observer list modifications
 309                                 //     -> for now, we assume that observers are not added/removed
 310                                 //        at runtime. otherwise we'd use sth. like an r/w lock with
 311                                 //        reader prioritization??
 312                                 for (l4_umword_t idx = 0; idx < _num_observers; ++idx) {
 313                                         //DEBUG() << "[" << i->id() << "] notifying: " << _observer_list[idx]->name();
 314                                         v = _observer_list[idx]->notify(i,t,tg,a);
 315                                         switch(v) {
 316                                                 case Romain::Observer::Finished:
 317                                                 case Romain::Observer::Replicatable:
 318                                                         //DEBUG() << "Fault handling done: " << std::hex << v;
 319                                                         return v;
 320                                                 default: break;
 321                                         }
 322                                 }
 323
 324                                 return v;
 325                         }
 326
 327
 328                         void query_observer_status()
 329                         {
 330                                 for (l4_umword_t idx = 0; idx < _num_observers; ++idx) {
 331                                         //INFO() << "querying: " << _observer_list[idx]->name();
 332                                         _observer_list[idx]->status();
 333                                 }
 334                         }
 335
 336
 337                         l4_umword_t instance_count() const { return _num_inst; }
 338
 339                         void logdump();
 340
 341                         void show_stats()
 342                         {
 343                                 INFO() << BOLD_PURPLE << "Halting threads..." << NOCOLOR;
 344                                 for (auto it = _threadgroups.begin(); it != _threadgroups.end(); ++it) {
 345                                         (*it)->halt();
 346                                 }
 347
 348                                 INFO() << BOLD_PURPLE << "-------- Observer statistics -------" << NOCOLOR;
 349                                 query_observer_status();
 350
 351                                 INFO() << BOLD_PURPLE << "-------- Replica statistics --------" << NOCOLOR;
 352                                 for (auto tg : _threadgroups) {
 353                                         INFO() << "Stats for thread group '" << tg->name << "'";
 354                                         for (auto t : tg->threads) {
 355                                                 t->print_stats();
 356                                         }
 357                                 }
 358
 359                                 logdump();
 360                         }
 361
 362                         Romain::App_instance* instance(l4_umword_t idx)
 363                         { return _instances[idx]; }
 364
 365                         static void VCPU_handler(Romain::InstanceManager *m, Romain::App_instance *i, Romain::App_thread *t, Romain::Thread_group* tg, Romain::App_model *a);
 366                         static void VCPU_startup(Romain::InstanceManager *m, Romain::App_instance *i, Romain::App_thread *t, Romain::Thread_group* tg, Romain::App_model *a);
 367
 368
 369                 private:
 370                         /*
 371                          * Read config from cfg file
 372                          */
 373                         void configure();
 374         };
 375
 376         extern Romain::InstanceManager* _the_instance_manager;
 377 }