kernel/fiasco/src/kern/per_cpu_data.cpp

   1 INTERFACE [mp]:
   2
   3 struct Per_cpu_ctor_data
   4 {
   5   void (*func)(void *, unsigned);
   6   void *base;
   7 };
   8
   9 #define DEFINE_PER_CPU_CTOR_UID(b) __per_cpu_ctor_ ## b
  10 #define DEFINE_PER_CPU_CTOR_DATA(id) \
  11   __attribute__((section(".bss.per_cpu_ctor_data"),used)) \
  12     static Per_cpu_ctor_data DEFINE_PER_CPU_CTOR_UID(id);
  13
  14 INTERFACE [!mp]:
  15
  16 #define DEFINE_PER_CPU_CTOR_DATA(id)
  17
  18 INTERFACE:
  19
  20 #include "static_init.h"
  21 #include "config.h"
  22 #include "context_base.h"
  23 #include <type_traits>
  24
  25 #define DEFINE_PER_CPU_P(p) \
  26   DEFINE_PER_CPU_CTOR_DATA(__COUNTER__) \
  27   __attribute__((section(".per_cpu.data"),init_priority(0xfffe - p)))
  28
  29 #define DEFINE_PER_CPU      DEFINE_PER_CPU_P(9)
  30 #define DEFINE_PER_CPU_LATE DEFINE_PER_CPU_P(19)
  31
  32 class Per_cpu_data
  33 {
  34 public:
  35   static void init_ctors();
  36   static void run_ctors(unsigned cpu);
  37   static void run_late_ctors(unsigned cpu);
  38   static bool valid(unsigned cpu);
  39 };
  40
  41 template< typename T > class Per_cpu_ptr;
  42
  43 template< typename T >
  44 class Per_cpu : private Per_cpu_data
  45 {
  46   friend class Per_cpu_ptr<T>;
  47 public:
  48   typedef T Type;
  49
  50   T const &cpu(unsigned) const;
  51   T &cpu(unsigned);
  52
  53   T const &current() const { return cpu(current_cpu()); }
  54   T &current() { return cpu(current_cpu()); }
  55
  56   Per_cpu();
  57   explicit Per_cpu(bool);
  58
  59   template<typename TEST>
  60   unsigned find_cpu(TEST const &test) const
  61   {
  62     for (unsigned i = 0; i < Config::Max_num_cpus; ++i)
  63       if (valid(i) && test(cpu(i)))
  64         return i;
  65
  66     return ~0U;
  67   }
  68
  69 private:
  70   T _d;
  71 };
  72
  73 template< typename T >
  74 class Per_cpu_ptr : private Per_cpu_data
  75 {
  76 public:
  77   typedef typename cxx::conditional<
  78     cxx::is_const<T>::value,
  79     Per_cpu<typename cxx::remove_cv<T>::type> const,
  80     Per_cpu<typename cxx::remove_cv<T>::type> >::type Per_cpu_type;
  81
  82   Per_cpu_ptr() {}
  83   Per_cpu_ptr(Per_cpu_type *o) : _p(&o->_d) {}
  84   Per_cpu_ptr &operator = (Per_cpu_type *o)
  85   {
  86     _p = &o->_d;
  87     return *this;
  88   }
  89
  90   T &cpu(unsigned cpu);
  91   T &current() { return cpu(current_cpu()); }
  92
  93 private:
  94   T *_p;
  95 };
  96
  97
  98 //---------------------------------------------------------------------------
  99 IMPLEMENTATION [!mp]:
 100
 101 #include <construction.h>
 102
 103 IMPLEMENT inline
 104 bool
 105 Per_cpu_data::valid(unsigned cpu)
 106 {
 107 #if defined NDEBUG
 108   (void)cpu;
 109   return 1;
 110 #else
 111   return cpu == 0;
 112 #endif
 113 }
 114
 115 IMPLEMENT inline
 116 template< typename T >
 117 T const &Per_cpu<T>::cpu(unsigned) const { return _d; }
 118
 119 IMPLEMENT inline
 120 template< typename T >
 121 T &Per_cpu<T>::cpu(unsigned) { return _d; }
 122
 123 IMPLEMENT
 124 template< typename T >
 125 Per_cpu<T>::Per_cpu()
 126 {}
 127
 128 IMPLEMENT
 129 template< typename T >
 130 Per_cpu<T>::Per_cpu(bool) : _d(0)
 131 {}
 132
 133 IMPLEMENT inline
 134 template< typename T >
 135 T &Per_cpu_ptr<T>::cpu(unsigned) { return *_p; }
 136
 137
 138 IMPLEMENT
 139 void
 140 Per_cpu_data::init_ctors()
 141 {
 142 }
 143
 144 IMPLEMENT inline
 145 void
 146 Per_cpu_data::run_ctors(unsigned)
 147 {
 148   extern ctor_function_t __PER_CPU_INIT_ARRAY_START__[];
 149   extern ctor_function_t __PER_CPU_INIT_ARRAY_END__[];
 150   run_ctor_functions(__PER_CPU_INIT_ARRAY_START__, __PER_CPU_INIT_ARRAY_END__);
 151
 152   extern ctor_function_t __PER_CPU_CTORS_LIST__[];
 153   extern ctor_function_t __PER_CPU_CTORS_END__[];
 154   run_ctor_functions(__PER_CPU_CTORS_LIST__, __PER_CPU_CTORS_END__);
 155 }
 156
 157 IMPLEMENT inline
 158 void
 159 Per_cpu_data::run_late_ctors(unsigned)
 160 {
 161   extern ctor_function_t __PER_CPU_LATE_INIT_ARRAY_START__[];
 162   extern ctor_function_t __PER_CPU_LATE_INIT_ARRAY_END__[];
 163   run_ctor_functions(__PER_CPU_LATE_INIT_ARRAY_START__,
 164                      __PER_CPU_LATE_INIT_ARRAY_END__);
 165
 166   extern ctor_function_t __PER_CPU_LATE_CTORS_LIST__[];
 167   extern ctor_function_t __PER_CPU_LATE_CTORS_END__[];
 168   run_ctor_functions(__PER_CPU_LATE_CTORS_LIST__, __PER_CPU_LATE_CTORS_END__);
 169 }
 170
 171
 172 //---------------------------------------------------------------------------
 173 INTERFACE [mp]:
 174
 175 #include <cstddef>
 176 #include <new>
 177
 178 #include "config.h"
 179
 180 EXTENSION
 181 class Per_cpu_data
 182 {
 183 private:
 184   typedef Per_cpu_ctor_data Ctor;
 185
 186   struct Ctor_vector
 187   {
 188   public:
 189     void push_back(void (*func)(void*,unsigned), void *base);
 190     unsigned len() const { return _len; }
 191     Ctor const &operator [] (unsigned idx) const
 192     {
 193       extern Ctor _per_cpu_ctor_data_start[];
 194       return _per_cpu_ctor_data_start[idx];
 195     }
 196
 197   private:
 198     unsigned _len;
 199   };
 200
 201 protected:
 202   enum { Num_cpus = Config::Max_num_cpus + 1 }; // add one for the never running CPU
 203   static long _offsets[Num_cpus] asm ("PER_CPU_OFFSETS");
 204   static unsigned late_ctor_start;
 205   static Ctor_vector ctors;
 206 };
 207
 208
 209 //---------------------------------------------------------------------------
 210 IMPLEMENTATION [mp]:
 211
 212 #include "panic.h"
 213 #include <construction.h>
 214 #include <cstring>
 215
 216 long Per_cpu_data::_offsets[Per_cpu_data::Num_cpus];
 217 unsigned Per_cpu_data::late_ctor_start;
 218 Per_cpu_data::Ctor_vector Per_cpu_data::ctors;
 219
 220 IMPLEMENT
 221 void
 222 Per_cpu_data::Ctor_vector::push_back(void (*func)(void*,unsigned), void *base)
 223 {
 224   extern Ctor _per_cpu_ctor_data_start[];
 225   extern Ctor _per_cpu_ctor_data_end[];
 226
 227   if (_per_cpu_ctor_data_start + _len >= _per_cpu_ctor_data_end)
 228     panic("out of per_cpu_ctor_space");
 229
 230   Ctor &c = _per_cpu_ctor_data_start[_len++];
 231   c.func = func;
 232   c.base = base;
 233 }
 234
 235
 236 IMPLEMENT inline
 237 bool
 238 Per_cpu_data::valid(unsigned cpu)
 239 { return cpu < Num_cpus && _offsets[cpu] != -1; }
 240
 241 IMPLEMENT inline template< typename T >
 242 T const &Per_cpu<T>::cpu(unsigned cpu) const
 243 { return *reinterpret_cast<T const *>((char  const *)&_d + _offsets[cpu]); }
 244
 245 IMPLEMENT inline template< typename T >
 246 T &Per_cpu<T>::cpu(unsigned cpu)
 247 { return *reinterpret_cast<T*>((char *)&_d + _offsets[cpu]); }
 248
 249 IMPLEMENT
 250 template< typename T >
 251 Per_cpu<T>::Per_cpu()
 252 {
 253   //printf("  Per_cpu<T>() [this=%p])\n", this);
 254   ctors.push_back(&ctor_wo_arg, this);
 255 }
 256
 257 IMPLEMENT
 258 template< typename T >
 259 Per_cpu<T>::Per_cpu(bool) : _d(0)
 260 {
 261   //printf("  Per_cpu<T>(bool) [this=%p])\n", this);
 262   ctors.push_back(&ctor_w_arg, this);
 263 }
 264
 265 PRIVATE static
 266 template< typename T >
 267 void Per_cpu<T>::ctor_wo_arg(void *obj, unsigned cpu)
 268 {
 269   //printf("Per_cpu<T>::ctor_wo_arg(obj=%p, cpu=%u -> %p)\n", obj, cpu, &(reinterpret_cast<Per_cpu<T>*>(obj)->cpu(cpu)));
 270   new (&reinterpret_cast<Per_cpu<T>*>(obj)->cpu(cpu)) T;
 271 }
 272
 273 PRIVATE static
 274 template< typename T >
 275 void Per_cpu<T>::ctor_w_arg(void *obj, unsigned cpu)
 276 {
 277   //printf("Per_cpu<T>::ctor_w_arg(obj=%p, cpu=%u -> %p)\n", obj, cpu, &reinterpret_cast<Per_cpu<T>*>(obj)->cpu(cpu));
 278   new (&reinterpret_cast<Per_cpu<T>*>(obj)->cpu(cpu)) T(cpu);
 279 }
 280
 281 IMPLEMENT inline
 282 template< typename T >
 283 T &Per_cpu_ptr<T>::cpu(unsigned cpu)
 284 { return *reinterpret_cast<T *>(reinterpret_cast<Address>(_p) + _offsets[cpu]); }
 285
 286 IMPLEMENT
 287 void
 288 Per_cpu_data::init_ctors()
 289 {
 290   for (unsigned i = 0; i < Num_cpus; ++i)
 291     _offsets[i] = -1;
 292 }
 293
 294 IMPLEMENT inline
 295 void
 296 Per_cpu_data::run_ctors(unsigned cpu)
 297 {
 298   extern ctor_function_t __PER_CPU_INIT_ARRAY_START__[];
 299   extern ctor_function_t __PER_CPU_INIT_ARRAY_END__[];
 300   extern ctor_function_t __PER_CPU_CTORS_LIST__[];
 301   extern ctor_function_t __PER_CPU_CTORS_END__[];
 302   if (cpu == 0)
 303     {
 304       run_ctor_functions(__PER_CPU_INIT_ARRAY_START__, __PER_CPU_INIT_ARRAY_END__);
 305       run_ctor_functions(__PER_CPU_CTORS_LIST__, __PER_CPU_CTORS_END__);
 306       late_ctor_start = ctors.len();
 307       return;
 308     }
 309
 310   for (unsigned i = 0; i < late_ctor_start; ++i)
 311     ctors[i].func(ctors[i].base, cpu);
 312 }
 313
 314 IMPLEMENT inline
 315 void
 316 Per_cpu_data::run_late_ctors(unsigned cpu)
 317 {
 318   extern ctor_function_t __PER_CPU_LATE_INIT_ARRAY_START__[];
 319   extern ctor_function_t __PER_CPU_LATE_INIT_ARRAY_END__[];
 320   extern ctor_function_t __PER_CPU_LATE_CTORS_LIST__[];
 321   extern ctor_function_t __PER_CPU_LATE_CTORS_END__[];
 322   if (cpu == 0)
 323     {
 324       run_ctor_functions(__PER_CPU_LATE_INIT_ARRAY_START__, __PER_CPU_LATE_INIT_ARRAY_END__);
 325       run_ctor_functions(__PER_CPU_LATE_CTORS_LIST__, __PER_CPU_LATE_CTORS_END__);
 326       return;
 327     }
 328
 329   unsigned c = ctors.len();
 330   for (unsigned i = late_ctor_start; i < c; ++i)
 331     ctors[i].func(ctors[i].base, cpu);
 332 }