// vim: ft=cpp

/*
 * manager --
 *
 *     Definition of the instance manager that knows about all
 *     redundant VCPUs as well as the fault observers.
 *
 * (c) 2011-2013 Björn Döbel <doebel@os.inf.tu-dresden.de>,
 *     economic rights: Technische Universität Dresden (Germany)
 * This file is part of TUD:OS and distributed under the terms of the
 * GNU General Public License 2.
 * Please see the COPYING-GPL-2 file for details.
 */

/*
 * Romain instance manager
 */

#pragma once

#include "constants.h"
#include "app"
#include "thread_group.h"
//#include "app_loading"
#include "fault_observers"
#include "cpuid.h"
#include "watchdog.h"

#include <vector>
#include <cstdio>
#include <algorithm>

#include "redundancy.h"
#include <l4/plr/measurements.h>

#if SPLIT_HANDLING
EXTERN_C void *split_handler_fn(void*);
#endif // SPLIT_HANDLING

namespace Romain
{
	/*
	 * Map logical (=sequential) CPU numbers to the topology of the underlying
	 * platform.
	 *
	 * Background: Fiasco.OC uses sequential CPU numbers to assign threads to CPUs. These
	 *             numbers are assigned during bootup and their order depends on the order
	 *             in which booted CPUs come up. This means, assigning two threads to CPUs
	 *             0 and 1 can in one case mean that they are running on the same core in
	 *             different hyperthreads, while in the next run, they run on the different
	 *             physical CPUs (or even NUMA domains, if we had those).
	 *
	 *             The purpose of this map is to make the assignment less surprising, e.g.,
	 *             for the same platform, assigning to two specific CPU IDs should always
	 *             lead to the same performance.
	 */
	class LogicalCPUMap
	{
		protected:
		std::vector<CPU_id> _fiasco_cpus;

		static l4_umword_t fiasco_count_online_cpus()
		{
			l4_umword_t ret = 0, max = 0;
			l4_sched_cpu_set_t cpus;
			chksys(L4Re::Env::env()->scheduler()->info(&max, &cpus), "CPU count failed");

			if (max > 31) {
				enter_kdebug("MAX > 31");
			}
			for (unsigned i = 0; i < max; ++i) {
				if (cpus.map & (1 << i))
					ret += (1 << cpus.granularity());
			}
			return ret;
		}

		static void* topology_scanner(void* data)
		{
			l4_debugger_set_object_name(pthread_l4_cap(pthread_self()), "romain::topscan");

			for (l4_umword_t cpu = 0; cpu < fiasco_count_online_cpus(); ++cpu) {
				Romain::LogicalCPUMap *m = reinterpret_cast<Romain::LogicalCPUMap*>(data);

				//INFO() << "CPU " << cpu;
				L4::Cap<L4::Thread> thread = Pthread::L4::cap(pthread_self());
				l4_sched_param_t sp = l4_sched_param(2);
				sp.affinity = l4_sched_cpu_set(cpu, 0);
				chksys(L4Re::Env::env()->scheduler()->run_thread(thread, sp));

				m->_fiasco_cpus[cpu] = CPUID::current_apicid(cpu);
			}
			return 0;
		}


		void scan_topology()
		{
			pthread_t p;
			l4_mword_t ret = pthread_create(&p, 0, Romain::LogicalCPUMap::topology_scanner, this);
			_check(ret != 0, "error creating topology scanner");
			ret = pthread_join(p, 0);

			for (l4_umword_t i = fiasco_count_online_cpus(); i < MAX_CPUS; ++i) {
				_fiasco_cpus[i].apic_id = 0xDEADBEEF;
			}

			INFO() << "Topology scan completed.";
		}


		void print_map()
		{
			INFO() << "Virtual to Fiasco CPU map";

			for (l4_umword_t i = 0; i < fiasco_count_online_cpus(); ++i) {
				INFO() << "virt# " << i << " fiasco# " << _fiasco_cpus[i].fiasco_id
				       << " apic# " << std::hex << _fiasco_cpus[i].apic_id;
			}
		}

		public:
			LogicalCPUMap()
				: _fiasco_cpus(MAX_CPUS)
			{
				/* Default: identity mapping if nothing else works */
				for (l4_umword_t i = 0; i < MAX_CPUS; ++i) {
					_fiasco_cpus[i].fiasco_id = i;
					_fiasco_cpus[i].apic_id   = i;
				}

				if (CPUID::have_cpuid() and
				    (CPUID::max_cpuid() >= CPUID::TOPOLOGY)) {

					scan_topology();
					auto f_it = _fiasco_cpus.begin();
					while ((*f_it).apic_id != 0xdeadbeef)
						++f_it;
					std::sort(_fiasco_cpus.begin(), f_it, CPU_id::comp_apic);
					// If Fiasco-CPU ID 0 is in the second half, reverse the order to
					// have the CPU0-Socket first
					// XXX Hack! This requires real socket knowledge.
					l4_umword_t logical0;
					for (logical0 = 0; logical0 < MAX_CPUS; ++logical0) {
						if (_fiasco_cpus[logical0].fiasco_id == 0)
							break;
					}
					if (logical0 >= CPUID::num_cpus() / 2) {
						std::reverse(_fiasco_cpus.begin(), f_it);
					}
					print_map();
				}

				//enter_kdebug("cpu");

			}

			l4_umword_t logicalToCPU(l4_umword_t log)
			{
				INFO() << log << " -> " << _fiasco_cpus[log].fiasco_id;
				return _fiasco_cpus[log].fiasco_id;
			}
	};

	/*
	 * Set up and keep track of the running instances
	 */
	class InstanceManager : public LogicalCPUMap
	{
		private:
			Romain::App_model *_am;
			std::vector<Romain::App_instance *> _instances;    // instance list
			std::vector<Romain::Thread_group *> _threadgroups; // list of thread groups

			Romain::Observer *_observer_list[Romain::MAX_OBSERVERS]; // exception observer list
			//Romain::RedundancyCallback *_callback;                 // the callback for redundancy checking
			l4_umword_t _num_observers;
			
			char const *_name;        // app name
			l4_umword_t    _num_inst;    // number of instances to run
			l4_umword_t _num_cpu;     // number of available CPUs

			l4_addr_t _init_eip;      // initial EIP (from ELF)
			l4_addr_t _init_esp;      // initial ESP (from ELF)
			l4_umword_t _gdt_min;        // start of GDT entries

			l4_umword_t     _argc;       // client argc
			char const **_argv;       // argv of the client

#if SPLIT_HANDLING
			pthread_t _split_handler; // resilient core handler thread
#endif // SPLIT_HANDLING

#if WATCHDOG
			bool  _watchdog_enable;
			int   _watchdog_timeout;

			Romain::Watchdog::SynchronizationMode _watchdog_mode;

			Romain::Watchdog* _watchdog;
#endif

			/*
			 * Prepare handler stack by pushing 3 values:
			 *
			 * 1st parameter is an app_thread
			 * 2nd parameter is an app_instance
			 * 3rd parameter is a dummy for the location where usually the
			 *                  return address would be located
			 * 4th parameter is the thread group we are dealing with
			 */
			l4_addr_t prepare_stack(l4_addr_t sp, Romain::App_instance *inst,
			                        Romain::App_thread *thread, Romain::Thread_group* tgroup);


			void configure_logflags(char *flags);
			void configure_fault_observers();
			void configure_redundancy();
			void configure_logbuf(l4_mword_t size);
			void configure_watchdog();

		public:
			InstanceManager(l4_umword_t argc, char const **argv, l4_umword_t num_instances = 1);


			/*
			 * Prepare the replica instances (VCPUs)
			 */
			void create_instances();

			/*
			 * Create a single replica thread along with
			 * its VCPU handler stack
			 */
			Romain::App_thread* create_thread(l4_umword_t eip, l4_umword_t esp,
											  l4_umword_t inst, Romain::Thread_group *group);

			/*
			 * Create a group of replica threads, e.g., all replicas
			 * of a single original thread
			 */
			Romain::Thread_group* create_thread_group(l4_umword_t eip, l4_umword_t esp,
													  std::string, l4_umword_t cap, l4_umword_t uid);

			/*
			 * Setup the VCPU handler stack.
			 *
			 * Puts fake parameters on the handler stack so that
			 * upon an event it looks like we got them passed as
			 * parameters.
			 */
			void prepare_handler_stack(Romain::App_thread *t, Romain::App_instance* i);

			/* 
			 * Start the instances
			 */
			void run_instances();


			void register_fault_observer(Romain::Observer* ob)
			{
				_check(_num_observers >= MAX_OBSERVERS, "no more observers");
				_observer_list[_num_observers++] = ob;
			}

#if WATCHDOG
			Romain::Watchdog* watchdog() { return _watchdog; }
#endif
#if 0
			Romain::RedundancyCallback *redundancy() const { return _callback; }


			void set_redundancy_callback(Romain::RedundancyCallback* cb)
			{
				_check(_callback != 0, "there is already a callback registered?");
				_callback = cb;
			}
#endif


			void startup_notify(Romain::App_instance *i, Romain::App_thread *t,
			                    Romain::Thread_group* tg, Romain::App_model *a) const
			{
				// XXX would need to be aware of observer list modifications
				//     -> for now, we assume that observers are not added/removed
				//        at runtime. otherwise we'd use sth. like an r/w lock with
				//        reader prioritization??
				for (l4_umword_t idx = 0; idx < _num_observers; ++idx) {
					DEBUG() << "[" << i->id() << "] startup notifying: " << _observer_list[idx]->name();
					_observer_list[idx]->startup_notify(i,t,tg,a);
				}
			}


			/*
			 * Entry point for CPU exception handling.
			 *
			 * Iterates over the list of registered observers and calls
			 * every observer's notify() function. Observers may stop
			 * this iteration process. See Romain::Observer::ObserverReturnVal
			 */
			Romain::Observer::ObserverReturnVal
			fault_notify(Romain::App_instance *i, Romain::App_thread *t,
			             Romain::Thread_group* tg, Romain::App_model *a)
			{
				Romain::Observer::ObserverReturnVal v = Romain::Observer::Ignored;
				// XXX would need to be aware of observer list modifications
				//     -> for now, we assume that observers are not added/removed
				//        at runtime. otherwise we'd use sth. like an r/w lock with
				//        reader prioritization??
				for (l4_umword_t idx = 0; idx < _num_observers; ++idx) {
					//DEBUG() << "[" << i->id() << "] notifying: " << _observer_list[idx]->name();
					v = _observer_list[idx]->notify(i,t,tg,a);
					switch(v) {
						case Romain::Observer::Finished:
						case Romain::Observer::Replicatable:
							//DEBUG() << "Fault handling done: " << std::hex << v;
							return v;
						default: break;
					}
				}

				return v;
			}


			void query_observer_status()
			{
				for (l4_umword_t idx = 0; idx < _num_observers; ++idx) {
					//INFO() << "querying: " << _observer_list[idx]->name();
					_observer_list[idx]->status();
				}
			}


			l4_umword_t instance_count() const { return _num_inst; }

			void logdump();

			void show_stats()
			{
				INFO() << BOLD_PURPLE << "Halting threads..." << NOCOLOR;
				for (auto it = _threadgroups.begin(); it != _threadgroups.end(); ++it) {
					(*it)->halt();
				}

				INFO() << BOLD_PURPLE << "-------- Observer statistics -------" << NOCOLOR;
				query_observer_status();

				INFO() << BOLD_PURPLE << "-------- Replica statistics --------" << NOCOLOR;
				for (auto tg : _threadgroups) {
					INFO() << "Stats for thread group '" << tg->name << "'";
					for (auto t : tg->threads) {
						t->print_stats();
					}
				}

				logdump();
			}

			Romain::App_instance* instance(l4_umword_t idx)
			{ return _instances[idx]; }

			static void VCPU_handler(Romain::InstanceManager *m, Romain::App_instance *i, Romain::App_thread *t, Romain::Thread_group* tg, Romain::App_model *a);
			static void VCPU_startup(Romain::InstanceManager *m, Romain::App_instance *i, Romain::App_thread *t, Romain::Thread_group* tg, Romain::App_model *a);


		private:
			/*
			 * Read config from cfg file
			 */
			void configure();
	};

	extern Romain::InstanceManager* _the_instance_manager;
}