fres/cbroker/fcb.c

   1 /**************************************************************************/
   2 /* ---------------------------------------------------------------------- */
   3 /* Copyright (C) 2006 - 2008 FRESCOR consortium partners:                 */
   4 /*                                                                        */
   5 /*   Universidad de Cantabria,              SPAIN                         */
   6 /*   University of York,                    UK                            */
   7 /*   Scuola Superiore Sant'Anna,            ITALY                         */
   8 /*   Kaiserslautern University,             GERMANY                       */
   9 /*   Univ. Politécnica  Valencia,           SPAIN                        */
  10 /*   Czech Technical University in Prague,  CZECH REPUBLIC                */
  11 /*   ENEA                                   SWEDEN                        */
  12 /*   Thales Communication S.A.              FRANCE                        */
  13 /*   Visual Tools S.A.                      SPAIN                         */
  14 /*   Rapita Systems Ltd                     UK                            */
  15 /*   Evidence                               ITALY                         */
  16 /*                                                                        */
  17 /*   See http://www.frescor.org for a link to partners' websites          */
  18 /*                                                                        */
  19 /*          FRESCOR project (FP6/2005/IST/5-034026) is funded             */
  20 /*       in part by the European Union Sixth Framework Programme          */
  21 /*       The European Union is not liable of any use that may be          */
  22 /*       made of this code.                                               */
  23 /*                                                                        */
  24 /*                                                                        */
  25 /*  This file is part of FRSH (FRescor ScHeduler)                         */
  26 /*                                                                        */
  27 /* FRSH is free software; you can redistribute it and/or modify it        */
  28 /* under terms of the GNU General Public License as published by the      */
  29 /* Free Software Foundation; either version 2, or (at your option) any    */
  30 /* later version.  FRSH is distributed in the hope that it will be        */
  31 /* useful, but WITHOUT ANY WARRANTY; without even the implied warranty    */
  32 /* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU    */
  33 /* General Public License for more details. You should have received a    */
  34 /* copy of the GNU General Public License along with FRSH; see file       */
  35 /* COPYING. If not, write to the Free Software Foundation, 675 Mass Ave,  */
  36 /* Cambridge, MA 02139, USA.                                              */
  37 /*                                                                        */
  38 /* As a special exception, including FRSH header files in a file,         */
  39 /* instantiating FRSH generics or templates, or linking other files       */
  40 /* with FRSH objects to produce an executable application, does not       */
  41 /* by itself cause the resulting executable application to be covered     */
  42 /* by the GNU General Public License. This exception does not             */
  43 /* however invalidate any other reasons why the executable file might be  */
  44 /* covered by the GNU Public License.                                     */
  45 /**************************************************************************/
  46
  47 /**
  48  * @file   fcb.c
  49  * @author Michal Sojka <sojkam1@fel.cvut.cz>
  50  * @date   Mon Oct 20 18:00:18 2008
  51  *
  52  * @brief  FRESCOR Contract Broker
  53  *
  54  *
  55  */
  56 #include <getopt.h>
  57 #include <forb.h>
  58 #include <forb/config.h>
  59 #include <fcb.h>
  60 #include <fcb_contact_info.h>
  61 #include <error.h>
  62 #include <errno.h>
  63 #include <stdio.h>
  64 #include <ul_gavlcust.h>
  65 #include <string.h>
  66 #include <ul_log.h>
  67 #include <ul_logreg.h>
  68 #include <forb/server_id.h>
  69 #include <fres_contract.h>
  70 #include "fcb_config.h"
  71 #include <fosa_clocks_and_timers.h>
  72 #include "contract_log.h"
  73 #if CONFIG_FCB_INET && !CONFIG_FORB_PROTO_INET_DEFAULT
  74 #include <forb/proto_inet.h>
  75 #endif
  76
  77 UL_LOG_CUST(ulogd_fcb);
  78 ul_log_domain_t ulogd_fcb = {UL_LOGL_MSG, "fcb"};
  79 UL_LOGREG_SINGLE_DOMAIN_INIT_FUNCTION(init_ulogd_fcb, ulogd_fcb);
  80
  81 fosa_abs_time_t start_time;
  82
  83
  84 /**
  85  * Resource identification
  86  */
  87 struct res_key {
  88         frsh_resource_type_t type;
  89         frsh_resource_id_t id;
  90 };
  91
  92 /**
  93  * Registered resource
  94  */
  95 struct resource {
  96         gavl_node_t node;
  97         struct res_key key;
  98         char *name;
  99         fres_resource_manager mng; /**< Object reference of the resource manager */
 100         gavl_cust_root_field_t allocators; /**< Registered allocators for this resource (from multiple applications/nodes) */
 101         ul_list_head_t sc_contracts; /**< Negotiated contracts with spare capacity for this resource */
 102 };
 103
 104 /**
 105  * Resource allocator registered in different nodes/applications
 106  */
 107 struct res_alloc {
 108         gavl_node_t node;
 109         forb_server_id app;
 110         fres_resource_allocator ra;
 111 };
 112
 113 struct fcb_contract {
 114         fres_contract_id_t id;
 115         struct res_alloc *ra;   /**< Allocator for this contract TODO: Remove contract if allocator is destroyed */
 116         gavl_node_t node_fcb;
 117         ul_list_node_t node_sc;
 118         ul_list_node_t node_reservation;
 119         struct {
 120                 int initial;
 121                 int try;
 122         } sc_variant;
 123         fosa_abs_time_t end_of_stability_period;
 124         struct fres_contract *user_contract; /* The contract sent by user. Set after sucessfull neotiation. */
 125         struct fres_contract *requested_contract; /* User request for the contract. Set by prepare_reservation_requests() in the beginning of negotiation. */
 126         struct fres_contract *to_be_reserved_contract;
 127         struct fres_contract *schedulable_contract;
 128 };
 129
 130 /**
 131  * Contract broker data
 132  */
 133 struct fcb {
 134         gavl_cust_root_field_t resources; /**< Registered resources */
 135         gavl_cust_root_field_t contracts; /**< Contracts negotiated by this FCB */
 136 };
 137
 138 /** List of contracts to be reserved during spare capacity rebalancing */
 139 struct reservation_list {
 140         ul_list_head_t fcb_contracts;
 141         unsigned length;
 142 };
 143
 144 struct fcb_contract *fcb_contract_new(fres_contract_id_t *id)
 145 {
 146         struct fcb_contract *fcb_contract;
 147
 148         fcb_contract = malloc(sizeof(*fcb_contract));
 149         if (!fcb_contract) {
 150                 return NULL;
 151         }
 152         memset(fcb_contract, 0, sizeof(*fcb_contract));
 153         fcb_contract->id = *id;
 154
 155         return fcb_contract;
 156 }
 157
 158 void fcb_contract_destroy(struct fcb_contract *fcb_contract)
 159 {
 160         if (fcb_contract->user_contract) {
 161                 fres_contract_destroy(fcb_contract->user_contract);
 162         }
 163         if (fcb_contract->to_be_reserved_contract) {
 164                 fres_contract_destroy(fcb_contract->to_be_reserved_contract);
 165         }
 166         if (fcb_contract->requested_contract) {
 167                 fres_contract_destroy(fcb_contract->requested_contract);
 168         }
 169         if (fcb_contract->schedulable_contract) {
 170                 fres_contract_destroy(fcb_contract->requested_contract);
 171         }
 172         free(fcb_contract);
 173 }
 174
 175 static inline int res_key_cmp(const struct res_key *a,
 176                               const struct res_key *b)
 177 {
 178         if (a->type < b->type) {
 179                 return -1;
 180         } else if (a->type > b->type) {
 181                 return +1;
 182         } else if (a->id < b->id) {
 183                 return -1;
 184         } else if (a->id > b->id) {
 185                 return +1;
 186         } else {
 187                 return 0;
 188         }
 189 }
 190
 191 /* Container for registered resources */
 192 GAVL_CUST_NODE_INT_DEC(fcb_resource   /* cust_prefix */,                \
 193                        struct fcb     /* cust_root_t */,                \
 194                        struct resource/* cust_item_t */,                \
 195                        struct res_key /* cust_key_t */,                 \
 196                        resources      /* cust_root_node */,             \
 197                        node           /* cust_item_node */,             \
 198                        key            /* cust_item_key */,              \
 199                        res_key_cmp    /* cust_cmp_fnc */);
 200
 201 GAVL_CUST_NODE_INT_IMP(fcb_resource   /* cust_prefix */,                \
 202                        struct fcb     /* cust_root_t */,                \
 203                        struct resource/* cust_item_t */,                \
 204                        struct res_key /* cust_key_t */,                 \
 205                        resources      /* cust_root_node */,             \
 206                        node           /* cust_item_node */,             \
 207                        key            /* cust_item_key */,              \
 208                        res_key_cmp    /* cust_cmp_fnc */);
 209
 210 /* Container for allocators registered for a given resource */
 211 GAVL_CUST_NODE_INT_DEC(fcb_alloc        /* cust_prefix */,              \
 212                        struct resource  /* cust_root_t */,              \
 213                        struct res_alloc /* cust_item_t */,              \
 214                        forb_server_id   /* cust_key_t */,               \
 215                        allocators       /* cust_root_node */,           \
 216                        node             /* cust_item_node */,           \
 217                        app              /* cust_item_key */,            \
 218                        fres_contract_id_cmp /* cust_cmp_fnc */);
 219
 220 GAVL_CUST_NODE_INT_IMP(fcb_alloc        /* cust_prefix */,              \
 221                        struct resource   /* cust_root_t */,             \
 222                        struct res_alloc /* cust_item_t */,              \
 223                        forb_server_id   /* cust_key_t */,               \
 224                        allocators       /* cust_root_node */,           \
 225                        node             /* cust_item_node */,           \
 226                        app              /* cust_item_key */,            \
 227                        forb_server_id_cmp /* cust_cmp_fnc */);
 228
 229 /* Container for contracts with spare capacity negotiated for a given resource */
 230 UL_LIST_CUST_DEC(sc_contracts        /* cust_prefix */,                 \
 231                  struct resource  /* cust_head_t */,                    \
 232                  struct fcb_contract /* cust_item_t */,                 \
 233                  sc_contracts   /* cust_head_field */,                  \
 234                  node_sc        /* cust_node_field */);
 235
 236 UL_LIST_CUST_DEC(reservation_list        /* cust_prefix */,             \
 237                  struct reservation_list  /* cust_head_t */,            \
 238                  struct fcb_contract /* cust_item_t */,                 \
 239                  fcb_contracts  /* cust_head_field */,                  \
 240                  node_reservation /* cust_node_field */);
 241
 242 /* Container for negotiated contracts */
 243 GAVL_CUST_NODE_INT_DEC(fcb_contract         /* cust_prefix */,          \
 244                        struct fcb           /* cust_root_t */,          \
 245                        struct fcb_contract  /* cust_item_t */,          \
 246                        fres_contract_id_t   /* cust_key_t */,           \
 247                        contracts            /* cust_root_node */,       \
 248                        node_fcb             /* cust_item_node */,       \
 249                        id                   /* cust_item_key */,        \
 250                        fres_contract_id_cmp /* cust_cmp_fnc */);
 251
 252 #if 1
 253 GAVL_CUST_NODE_INT_IMP(fcb_contract         /* cust_prefix */,          \
 254                        struct fcb           /* cust_root_t */,          \
 255                        struct fcb_contract  /* cust_item_t */,          \
 256                        fres_contract_id_t   /* cust_key_t */,           \
 257                        contracts            /* cust_root_node */,       \
 258                        node_fcb             /* cust_item_node */,       \
 259                        id                   /* cust_item_key */,        \
 260                        fres_contract_id_cmp /* cust_cmp_fnc */);
 261 #else
 262 #include "fcb_contract_gavl.inc"
 263 #endif
 264
 265
 266 #define o2fcb(o) (struct fcb*)forb_instance_data(o)
 267
 268 struct res_key*
 269 get_res_key(const struct fres_contract *contract, struct res_key *key)
 270 {
 271         fres_block_resource *block_res;
 272
 273         block_res = fres_contract_get_resource(contract);
 274         if (!block_res) {
 275                 ul_logerr("No resource specified\n");
 276                 return NULL;
 277         } else {
 278                 key->type = block_res->resource_type;
 279                 key->id = block_res->resource_id;
 280         }
 281         return key;
 282 }
 283
 284 /**
 285  * Fills in an array of fcb_contracts according to requests submited
 286  * to negotiate_contracts(). For every contract in @a contracts, there
 287  * is one fcb_contract in @a fcb_contracts array.
 288  *
 289  * @param fcb
 290  * @param fcb_contracts Where to store pointers to fcb_contracts
 291  * @param resource Resource for which negotiation is being done.
 292  * @param app Application which requests negotiation
 293  * @param contracts Contracts submited to negotiate_contracts()
 294  * @param num Number of contracts in contracts (which is the same as the number in fcb_contracts).
 295  *
 296  * @return Zero on success, non-zero error code on error.
 297  */
 298 static int
 299 prepare_fcb_contracts(struct fcb *fcb, struct fcb_contract *fcb_contracts[],
 300                       struct resource **resource,
 301                       forb_server_id *app,
 302                       struct fres_contract *contracts[], int num)
 303 {
 304         unsigned i;
 305         struct fcb_contract *fc;
 306         struct res_alloc *ra;
 307         struct res_key key, key2 = {-1,-1};
 308
 309         for (i=0; i<num; i++) {
 310                 struct fres_contract *c = contracts[i];
 311
 312                 log_contract("Received contract", i, c);
 313
 314                 if (fres_contract_id_is_empty(&c->id)) {
 315                         /* Normal negotiation request */
 316                         forb_uuid_generate((forb_uuid_t *)&c->id);
 317                         fc = fcb_contract_new(&c->id);
 318                         if (!fc)
 319                                 return errno;
 320                         fcb_contracts[i] = fc;
 321                         if (!get_res_key(c, &key)) {
 322                                 return FRSH_ERR_RESOURCE_ID_INVALID;
 323                         }
 324                 } else {
 325                         fc = fcb_contract_find(fcb, &c->id);
 326                         if (!fc) {
 327                                 char str[60];
 328                                 fres_contract_id_to_string(str, &c->id, sizeof(str));
 329                                 ul_logerr("Attempt to change unknown contract %s\n", str);
 330                                 return FRES_ERR_NOTHING_TO_RENEGOTIATE;
 331                         } else {
 332                                 fcb_contracts[i] = fc;
 333                                 if (fres_contract_get_num_blocks(c) == 0) {
 334                                         /* Cancelation */
 335                                         get_res_key(fc->user_contract, &key);
 336                                 } else {
 337                                         /* Renegotiation */
 338                                         if (!get_res_key(c, &key)) {
 339                                                 return FRSH_ERR_RESOURCE_ID_INVALID;
 340                                         }
 341                                 }
 342                         }
 343                 }
 344                 fc->requested_contract = c;
 345
 346                 /* Check that all contracts are for the same resource */
 347                 if (i==0) {
 348                         key2 = key;
 349                         *resource = fcb_resource_find(fcb, &key);
 350                         if (!*resource) {
 351                                 ul_logerr("No resource manager for %d.%d registered\n",
 352                                           key.type, key.id);
 353                                 return FRES_ERR_NO_RESOURCE_MANAGER;
 354                         }
 355                 }
 356                 else if (key.type != key2.type ||
 357                          key.id   != key2.id) {
 358                         ul_logerr("Contract #%d differs in resource!\n", i);
 359                         return FRSH_ERR_RESOURCE_ID_INVALID;
 360                 }
 361
 362                 /* Find allocator for this request */
 363                 ra = fcb_alloc_find(*resource, app);
 364                 if (!ra) {
 365                         char str[60];
 366                         forb_server_id_to_string(str, app, sizeof(str));
 367                         ul_logerr("No resource allocator found for %d.%d and %s\n",
 368                                   (*resource)->key.type, (*resource)->key.id, str);
 369                         return FRES_ERR_NO_RESOURCE_ALLOCATOR;
 370                 }
 371                 fc->ra = ra;
 372         }
 373         return 0;
 374 }
 375
 376 /**
 377  * @param resource Resource for which to rebalance capacity and negotiate new contracts
 378  * @param fcb_contract New requests to negotiate
 379  * @param num The number of elements in @a fcb_contract
 380  * @param[out] rl List with changed contracts to be commited
 381  */
 382 static void
 383 prepare_reservation_list(struct resource *resource,
 384                          struct fcb_contract *fcb_contract[], int num,
 385                          struct reservation_list *rl)
 386 {
 387         int i;
 388         fosa_abs_time_t now;
 389         struct fcb_contract *fc;
 390
 391         reservation_list_init_head(rl);
 392         rl->length = 0;
 393         for (i=0; i<num; i++) {
 394                 assert(fcb_contract[i]->requested_contract != NULL);
 395                 reservation_list_insert(rl, fcb_contract[i]);
 396                 rl->length++;
 397         }
 398         fosa_clock_get_time(CLOCK_REALTIME, &now);
 399         ul_list_for_each(sc_contracts, resource, fc) {
 400                 if (fosa_abs_time_smaller(fc->end_of_stability_period, now) &&
 401                     fc->requested_contract == NULL) /* Do not insert contract inserted above */
 402                 {
 403                         reservation_list_insert(rl, fc);
 404                         rl->length++;
 405                 }
 406         }
 407 }
 408
 409 /**
 410  *
 411  *
 412  * @param resource Resource for which to rebalance capacity and negotiate new contracts
 413  * @param rl List with changed contracts to be commited
 414  *
 415  * @return Zero on success, non-zero error code on error
 416  */
 417 static int
 418 rebalance_spare_capacity_and_reserve(struct resource *resource,
 419                                      struct reservation_list *rl)
 420 {
 421         int ret;
 422         unsigned i;
 423         struct fcb_contract *fc;
 424         fres_block_spare_capacity *s;
 425
 426         fres_contract_ptr_seq contracts;
 427         if (!forb_sequence_alloc_buf(contracts, rl->length)) {
 428                 return errno;
 429         }
 430         contracts._length = rl->length;
 431         i=0;
 432         /* Initialize optimization */
 433         ul_list_for_each(reservation_list, rl, fc) {
 434                 fc->to_be_reserved_contract =
 435                         fc->requested_contract ? fres_contract_duplicate(fc->requested_contract) :
 436                         fc->user_contract ? fres_contract_duplicate(fc->user_contract) :
 437                         NULL;
 438                 assert(fc->to_be_reserved_contract != NULL);
 439
 440                 forb_sequence_elem(contracts, i) = fc->to_be_reserved_contract;
 441                 i++;
 442
 443                 s = fres_contract_get_spare_capacity(fc->to_be_reserved_contract);
 444                 if (s && s->granularity == FRSH_GR_DISCRETE) {
 445                         fc->sc_variant.initial = s->variants._length - 1;
 446                         fc->sc_variant.try = fc->sc_variant.initial;
 447                 }
 448         }
 449
 450         bool all_combinations_tried;
 451         int criterion, best_criterion = -1;
 452         struct fcb_contract *fcb_changed;
 453         /* Exhaustive search. Try all combinations of spare capacity
 454          * variants and find the one with best creterion. */
 455         do {
 456                 all_combinations_tried = true;
 457                 fcb_changed = NULL;
 458                 criterion = 0;
 459                 /* Prepare spare capacity variant */
 460                 ul_list_for_each(reservation_list, rl, fc) {
 461                         s = fres_contract_get_spare_capacity(fc->to_be_reserved_contract);
 462                         /* TODO: Simulate continuous granularity by discretization */
 463                         if (s && s->granularity == FRSH_GR_DISCRETE) {
 464                                 fres_container_copy(fc->to_be_reserved_contract->container,
 465                                                     forb_sequence_elem(s->variants, fc->sc_variant.try));
 466                                 criterion += fc->sc_variant.try;
 467
 468                                 if (fcb_changed == NULL) {
 469                                         /* Chnage tried variant for the next round */
 470                                         fc->sc_variant.try = fc->sc_variant.try > 0 ?
 471                                                 fc->sc_variant.try - 1 :
 472                                                 s->variants._length - 1;
 473                                         /* Do not change other
 474                                          * contracts unless we have
 475                                          * tried all variants */
 476                                         if (fc->sc_variant.try != fc->sc_variant.initial) {
 477                                                 fcb_changed = fc;
 478                                         }
 479                                 }
 480                                 if (fc->sc_variant.try != fc->sc_variant.initial)
 481                                         all_combinations_tried = false;
 482                         }
 483                 }
 484
 485                 if (criterion > best_criterion) {
 486                         CORBA_Environment ev;
 487                         /* Reserve contract */
 488                         ret = fres_resource_manager_reserve_contracts(resource->mng, &contracts, &ev);
 489                         if (forb_exception_occurred(&ev)) {
 490                                 ret = fres_forbex2err(&ev);
 491                                 ul_logerr("FORB exception when reserving contracts\n");
 492                                 goto err;
 493                         }
 494                         if (ret < 0) {
 495                                 ul_logerr("Contract reservation error %d\n", ret);
 496                                 ret = FRES_ERR_ADMISSION_TEST;
 497                                 goto err;
 498                         }
 499                         if (ret == 0) { /* negotiation succeeded */
 500                                 best_criterion = criterion;
 501                         }
 502                 }
 503         } while (!all_combinations_tried);
 504
 505         if (best_criterion == -1) {
 506                 ret = FRSH_ERR_CONTRACT_REJECTED;
 507         } else {
 508                 /* At least some variant succeeded */
 509                 ret = 0;
 510                 sc_contracts_init_head(resource);
 511                 ul_list_for_each(reservation_list, rl, fc) {
 512                         s = fres_contract_get_spare_capacity(fc->to_be_reserved_contract);
 513                         if (s && s->granularity == FRSH_GR_DISCRETE) {
 514                                 sc_contracts_insert(resource, fc);
 515                         }
 516                 }
 517         }
 518 err:
 519         ul_list_for_each(reservation_list, rl, fc) {
 520                 fres_contract_destroy(fc->to_be_reserved_contract);
 521                 fc->to_be_reserved_contract = NULL;
 522         }
 523         forb_sequence_free_buf(contracts, forb_no_destructor);
 524         return ret;
 525 }
 526
 527 /**
 528  * Create/change VReses according to @a schedulable_contracts.
 529  *
 530  * There might be more allocators for schedulable contracts, so merge
 531  * adjacent vreses with the same allocator together and create/change
 532  * them in one step. Also the order of schedulable contracts might be
 533  * different from the initial order od ids in commit_contracts()
 534  * therefore we have to search for every contract.
 535  */
 536 static int
 537 change_vreses(struct fcb *fcb, fres_contract_ptr_seq *schedulable_contracts)
 538 {
 539         struct res_alloc *last_ra = NULL;
 540         fres_contract_ptr_seq vreses;
 541         int i, ret = 0;
 542         CORBA_Environment ev;
 543
 544         if (!forb_sequence_alloc_buf(vreses, schedulable_contracts->_length)) {
 545                 return errno;
 546         }
 547         vreses._length = 0;
 548
 549         for (i=0; i<schedulable_contracts->_length; i++) {
 550                 struct fcb_contract *fc;
 551                 struct fres_contract *sc = schedulable_contracts->_buffer[i];
 552
 553                 log_contract("Changing VRES", i, sc);
 554
 555                 fc = fcb_contract_find(fcb, &sc->id);
 556                 assert(fc != NULL);
 557
 558                 if (true /* TODO: if the schedulable contract is changed */ &&
 559                     fres_contract_get_type(&sc) != FRSH_CT_DUMMY) {
 560                         if (last_ra != fc->ra) {
 561                                 if (vreses._length) {
 562                                         ret = fres_resource_allocator_change_vreses(last_ra->ra,
 563                                                                                     &vreses, &ev);
 564                                         if (forb_exception_occurred(&ev)) {
 565                                                 ret = fres_forbex2err(&ev);
 566                                                 goto err;
 567                                         }
 568                                         if (ret) goto err;
 569                                 }
 570                                 vreses._length = 0;
 571                         }
 572                         vreses._buffer[vreses._length] = sc;
 573                         vreses._length++;
 574                         fres_contract_destroy(fc->schedulable_contract);
 575                         fc->schedulable_contract = fres_contract_duplicate(sc);
 576                         last_ra = fc->ra;
 577
 578                 }
 579         }
 580         if (last_ra) {
 581                 ret = fres_resource_allocator_change_vreses(last_ra->ra,
 582                                                             &vreses, &ev);
 583                 if (forb_exception_occurred(&ev))
 584                         ret = fres_forbex2err(&ev);
 585         }
 586 err:
 587         forb_sequence_free_buf(vreses, forb_no_destructor);
 588         return ret;
 589 }
 590
 591 CORBA_long
 592 negotiate_contracts(fres_contract_broker obj,
 593                     const fres_contract_ptr_seq* contracts,
 594                     fres_contract_id_seq** ids_out,
 595                     CORBA_Environment *ev)
 596 {
 597         struct fcb *fcb = o2fcb(obj);
 598         struct resource *resource = NULL;
 599         int ret = 0;
 600         forb_server_id app;
 601         fres_contract_ptr_seq *schedulable_contracts;
 602         struct fcb_contract **fcb_contracts, *fc;
 603         unsigned i;
 604         fres_contract_id_seq commit_ids;
 605
 606         /* Prepare output sequence for the case we return eariler with
 607          * an error */
 608         forb_sequence_alloc(*ids_out, 0);
 609         if (!*ids_out) {
 610                 ev->major = FORB_EX_NO_MEMORY;
 611                 goto err;
 612         }
 613         CORBA_sequence_set_release(*ids_out, CORBA_TRUE);
 614
 615         forb_get_req_source(obj, &app);
 616
 617         fcb_contracts = malloc(sizeof(fcb_contracts[0])*contracts->_length);
 618         if (!fcb_contracts) {
 619                 ret = errno;
 620                 goto err;
 621         }
 622         memset(fcb_contracts, 0, sizeof(fcb_contracts[0])*contracts->_length);
 623
 624         ret = prepare_fcb_contracts(fcb, fcb_contracts, &resource,
 625                                     &app, contracts->_buffer, contracts->_length);
 626         if (ret)
 627                 goto err_free_fcb_contracts;
 628
 629         struct reservation_list rl;
 630         prepare_reservation_list(resource,
 631                                  fcb_contracts, contracts->_length,
 632                                  &rl);
 633
 634         /* Allocate all the needed memory before doing reservation. If
 635          * there is no enough memory, it has no sense to call resource
 636          * manager. */
 637         if (!forb_sequence_alloc_buf(commit_ids, rl.length)) {
 638                 ret = errno;
 639                 goto err_free_fcb_contracts;
 640         }
 641
 642         /* Reserve contracts */
 643         ret = rebalance_spare_capacity_and_reserve(resource, &rl);
 644         if (ret) {
 645                 if (ret == FRSH_ERR_CONTRACT_REJECTED) {
 646                         ul_logmsg("Contract(s) was/were rejected\n");
 647                 } else {
 648                         char msg[100];
 649                         fres_strerror(ret, msg, sizeof(msg));
 650                         ul_logerr("Reservation error: %s\n", msg);
 651                 }
 652                 goto err_free_fcb_contracts;
 653         }
 654
 655         /* Commit contracts */
 656         commit_ids._length = rl.length;
 657         i=0;
 658         ul_list_for_each(reservation_list, &rl, fc) {
 659                 forb_sequence_elem(commit_ids, i) = fc->id;
 660                 i++;
 661         }
 662
 663         fres_resource_manager_commit_contracts(resource->mng, &commit_ids,
 664                                                &schedulable_contracts, ev);
 665         if (forb_exception_occurred(ev)) {
 666                 ret = fres_forbex2err(ev);
 667                 goto err_cancel_reservation;
 668         }
 669
 670         /* Add new contracts to our fcb database for later
 671          * reference. Canceled contracts are removed below. */
 672         for (i=0; i<contracts->_length; i++) {
 673                 fc =  fcb_contracts[i];
 674
 675                 if (fc->user_contract) {
 676                         if (fres_contract_get_num_blocks(fc->requested_contract) > 0) {
 677                                 /* Renegotiation */
 678                                 fres_contract_destroy(fc->user_contract);
 679                                 fc->user_contract = fres_contract_duplicate(fc->requested_contract);
 680                                 /* Note: requested_contract is also
 681                                  * pointed by contracts parameter and
 682                                  * will be freed by FORB. */
 683                                 fc->requested_contract = NULL;
 684                         }
 685                 } else {
 686                         /* Insert new contracts */
 687                         fcb_contract_insert(fcb, fcb_contracts[i]);
 688                         fc->user_contract = fres_contract_duplicate(fc->requested_contract);
 689                         fc->requested_contract = NULL;
 690                         /* See the note above. */
 691                 }
 692         }
 693
 694         ret = change_vreses(fcb, schedulable_contracts);
 695         if (ret)
 696                 goto err_cancel_reservation;
 697
 698         forb_sequence_free(schedulable_contracts, fres_contract_ptr_destroy);
 699         if (ret != 0) {
 700                 ul_logerr("VRes was not created\n");
 701                 goto err_cancel_contracts;
 702         }
 703
 704
 705         /* Return IDs and delete canceled contracts from out database */
 706         if (!forb_sequence_alloc_buf(**ids_out, contracts->_length)) {
 707                 ev->major = FORB_EX_NO_MEMORY;
 708                 goto err_cancel_contracts;
 709         }
 710         (*ids_out)->_length = contracts->_length;
 711         for (i=0; i<contracts->_length; i++) {
 712                 struct fcb_contract *fc =  fcb_contracts[i];
 713                 forb_sequence_elem(**ids_out, i) = fc->id;
 714
 715                 if (fc->requested_contract &&
 716                     fres_contract_get_num_blocks(fc->requested_contract) == 0) {
 717                         fcb_contract_delete(fcb, fc);
 718                         /* Note: requested_contract is also pointed by
 719                          * contracts parameter and will be freed by FORB. */
 720                         fc->requested_contract = NULL;
 721                         fcb_contract_destroy(fc);
 722                 }
 723         }
 724         return 0;
 725
 726 err_cancel_contracts:
 727         /* TODO */
 728         goto err_free_fcb_contracts;
 729 err_cancel_reservation:
 730         fres_resource_manager_cancel_reservations(resource->mng, &commit_ids, ev);
 731 err_free_fcb_contracts:
 732         for (i=0; i<contracts->_length; i++) {
 733                 fc = fcb_contracts[i];
 734                 if (fc && !fc->user_contract) {
 735                         fcb_contracts[i]->requested_contract = NULL; /* Destroyed by FORB */
 736                         fcb_contract_destroy(fcb_contracts[i]);
 737                 }
 738         }
 739         free(fcb_contracts);
 740 err:
 741         return ret;
 742 }
 743
 744 void redistribute_spare_capacity(fres_contract_broker obj,
 745                                  const frsh_resource_type_t restype,
 746                                  const frsh_resource_id_t resid,
 747                                  CORBA_Environment *ev)
 748 {
 749         struct fcb *fcb = o2fcb(obj);
 750         struct res_key key = {restype, resid };
 751         struct resource *resource;
 752         struct reservation_list rl;
 753
 754         resource = fcb_resource_find(fcb, &key);
 755
 756         prepare_reservation_list(resource, NULL, 0, &rl);
 757
 758 /*      forb_sequence_alloc(ids, rl.length); */
 759 /*      if (!ids || !ids->_buffer) { */
 760 /*              ev->major = FORB_EX_NO_MEMORY; */
 761 /*              goto err_free_fcb_contracts; */
 762 /*      } */
 763 /*      CORBA_sequence_set_release(ids, CORBA_TRUE); */
 764 /*      *ids_out = ids;         /\* ids is freed by FORB *\/ */
 765
 766
 767         rebalance_spare_capacity_and_reserve(resource, &rl);
 768         /* Commit */
 769 }
 770
 771 CORBA_long register_resource(fres_contract_broker obj,
 772                             const frsh_resource_type_t restype,
 773                             const frsh_resource_id_t resid,
 774                             const fres_resource_desc *desc,
 775                             CORBA_Environment *ev)
 776 {
 777         struct fcb *fcb = o2fcb(obj);
 778         struct resource *res, *res2;
 779
 780         res = malloc(sizeof(*res));
 781         if (!res) goto err;
 782         memset(res, 0, sizeof(*res));
 783         res->key.type = restype;
 784         res->key.id = resid;
 785         res2 = fcb_resource_find(fcb, &res->key);
 786         if (res2) {
 787                 if (forb_object_is_stale(res2->mng)) {
 788                         ul_logmsg("Removing stale manager for resource %d.%d\n",
 789                                   restype, resid);
 790                         forb_object_release(res2->mng);
 791                         fcb_resource_delete(fcb, res2);
 792                         /* TODO: Delete also all allocators associated
 793                          * with this stale resource manager. */
 794                         free(res);
 795                         res = res2;
 796                 } else {
 797                         ul_logerr("Resource manager %d.%d already registered\n",
 798                                   restype, resid);
 799                         goto free_err;
 800                 }
 801         }
 802         res->mng = forb_object_duplicate(desc->manager);
 803         res->name = desc->name;
 804
 805         fcb_alloc_init_root_field(res);
 806         sc_contracts_init_head(res);
 807         ul_logmsg("Registering manager for resource \"%s\" (%d.%d)\n",
 808                   res->name, restype, resid);
 809         fcb_resource_insert(fcb, res);
 810         return 0;
 811 free_err:
 812         free(res);
 813 err:
 814         return -1;
 815 }
 816
 817
 818 CORBA_long register_allocator(fres_contract_broker obj,
 819                               const frsh_resource_type_t restype,
 820                               const frsh_resource_id_t resid,
 821                               const fres_resource_allocator ra_obj,
 822                               CORBA_Environment *ev)
 823 {
 824         struct fcb *fcb = o2fcb(obj);
 825         struct resource *res;
 826         struct res_alloc *ra;
 827         struct res_key resource;
 828         forb_server_id server_id;
 829         char server_id_str[40];
 830         int ret;
 831
 832         forb_get_server_id(ra_obj, &server_id);
 833         forb_server_id_to_string(server_id_str, &server_id, sizeof(server_id_str));
 834         ul_logmsg("Registering allocator for resource %d.%d in app %s\n",
 835                   restype, resid, server_id_str);
 836
 837         resource.type = restype;
 838         resource.id = resid;
 839         res = fcb_resource_find(fcb, &resource);
 840         if (!res) {
 841                 ul_logerr("No manager found for %d.%d. Unable to register the allocator!\n",
 842                           restype, resid);
 843                 ret = FRES_ERR_NO_RESOURCE_MANAGER;
 844                 goto err;
 845         }
 846         ra = fcb_alloc_find(res, &server_id);
 847         if (ra) {
 848                 char *str = forb_object_to_string(ra_obj);
 849                 ul_logerr("Allocator from already registered (%s)\n",
 850                           str);
 851                 forb_free(str);
 852                 ret = FRES_ERR_ALLOCATOR_ALREADY_REGISTERED;
 853                 goto err;
 854         }
 855         ra = malloc(sizeof(*ra));
 856         if (!ra) {
 857                 ret = ENOMEM;
 858                 goto err;
 859         }
 860         ra->app = server_id;
 861         ra->ra = forb_object_duplicate(ra_obj);
 862         fcb_alloc_insert(res, ra);
 863         return 0;
 864 err:
 865         return ret;
 866 }
 867
 868 void get_resources(fres_contract_broker obj, fres_resource_seq** resources, CORBA_Environment *ev)
 869 {
 870         struct fcb *fcb = o2fcb(obj);
 871         fres_resource_seq *seq;
 872         struct resource *res;
 873         int n;
 874
 875         seq = malloc(sizeof(*seq));
 876         if (!seq) {
 877                 ev->major = FORB_EX_NO_MEMORY;
 878                 return;
 879         }
 880         memset(seq, 0, sizeof(*seq));
 881
 882         n=0;
 883         gavl_cust_for_each(fcb_resource, fcb, res) {
 884                 n++;
 885         }
 886
 887         seq->_buffer = CORBA_sequence_fres_resource_allocbuf(n);
 888         seq->_maximum = n;
 889         seq->_length = n;
 890
 891         n = 0;
 892         gavl_cust_for_each(fcb_resource, fcb, res) {
 893                 seq->_buffer[n].restype = res->key.type;
 894                 seq->_buffer[n].resid = res->key.id;
 895                 seq->_buffer[n].desc.manager = res->mng;
 896                 seq->_buffer[n].desc.name = res->name;
 897                 n++;
 898         }
 899
 900         *resources = seq;
 901 }
 902
 903 #if CONFIG_FCB_INET && !CONFIG_FORB_PROTO_INET_DEFAULT
 904 static int register_inet_port(forb_orb orb)
 905 {
 906         forb_port_t *port = forb_malloc(sizeof(*port));
 907         int ret;
 908         struct in_addr listen_on;
 909
 910         if (!port)
 911                 return -1;
 912         memset(port, 0, sizeof(*port));
 913         listen_on.s_addr = INADDR_ANY;
 914         ret = forb_inet_port_init(&port->desc, listen_on, 0);
 915         if (ret)
 916                 return ret;
 917         ret = forb_register_port(orb, port);
 918         return ret;
 919 }
 920 #endif
 921
 922 struct forb_fres_contract_broker_impl impl = {
 923         .negotiate_contracts = negotiate_contracts,
 924         .register_resource = register_resource,
 925         .register_allocator = register_allocator,
 926         .redistribute_spare_capacity = redistribute_spare_capacity,
 927         .get_resources = get_resources,
 928 };
 929
 930 void peer_discovery_callback(const forb_orb peer_orb, const char *orb_id)
 931 {
 932         forb_server_id server_id;
 933         char server_id_str[sizeof(forb_server_id)*2+1];
 934
 935         forb_get_server_id(peer_orb, &server_id);
 936         forb_server_id_to_string(server_id_str, &server_id, sizeof(server_id_str));
 937 #if 0
 938         ul_logmsg("peer discovered: %s (orb_id '%s')\n", server_id_str, orb_id);
 939 #endif
 940
 941         if (orb_id == NULL)
 942                 return;
 943
 944         if (strcmp(orb_id, "org.frescor.fcb") == 0) {
 945                 fosa_abs_time_t now;
 946                 fosa_rel_time_t delay;
 947                 fosa_clock_get_time(CLOCK_REALTIME, &now);
 948                 delay = fosa_abs_time_extract_interval(start_time, now);
 949                 ul_logmsg("node joined: %s (time %ld ms)\n", server_id_str,
 950                           fosa_rel_time_to_msec(delay));
 951         }
 952 }
 953
 954 void peer_dead_callback(const forb_orb peer_orb, const char *orb_id)
 955 {
 956 }
 957
 958 static struct option long_opts[] = {
 959     { "loglevel", 1, 0, 'l' },
 960     { 0, 0, 0, 0}
 961 };
 962
 963 static void
 964 usage(void)
 965 {
 966         printf("usage: fcb [ options ]\n");
 967         printf("  -l, --loglevel <number>|<domain>=<number>,...\n");
 968 }
 969
 970 int print_log_domain(ul_log_domain_t *domain, void *context)
 971 {
 972         printf("%s = %d\n", domain->name, domain->level);
 973         return 0;
 974 }
 975
 976 int main(int argc, char *argv[])
 977 {
 978         forb_orb orb;
 979         struct fcb fcb_data;
 980         fres_contract_broker fcb;
 981         forb_executor_t executor;
 982         int ret;
 983         forb_init_attr_t attr = {
 984                 .orb_id = "org.frescor.fcb",
 985                 .peer_discovery_callback = peer_discovery_callback,
 986                 .peer_dead_callback = peer_dead_callback,
 987                 .fixed_tcp_port = FCB_TCP_PORT,
 988 #ifdef CONFIG_FORB_PROTO_INET_DEFAULT
 989                 .fixed_server_id = FCB_SERVER_ID,
 990                 .redistribute_hellos = true,
 991 #endif
 992         };
 993         int  opt;
 994
 995         while ((opt = getopt_long(argc, argv, "l:", &long_opts[0], NULL)) != EOF) {
 996                 switch (opt) {
 997                         case 'l':
 998                                 if (*optarg == '?') {
 999                                         ul_logreg_for_each_domain(print_log_domain, NULL);
1000                                         exit(0);
1001                                 }
1002                                 {
1003                                         int ret;
1004                                         ret = ul_log_domain_arg2levels(optarg);
1005                                         if (ret)
1006                                                 error(1, EINVAL, "Error parsing -l argument at char %d\n", ret);
1007                                 }
1008                                 break;
1009                         case 'h':
1010                         /*default:*/
1011                                 usage();
1012                                 exit(opt == 'h' ? 0 : 1);
1013                 }
1014         }
1015
1016         fosa_clock_get_time(CLOCK_REALTIME, &start_time);
1017
1018         orb = forb_init(&argc, &argv, &attr);
1019         if (!orb) error(1, errno, "FORB initialization failed");
1020
1021 #if CONFIG_FCB_INET && !CONFIG_FORB_PROTO_INET_DEFAULT
1022         ret = register_inet_port(orb);
1023         if (ret) error(0, errno, "INET port registration failed");
1024 #endif
1025
1026         fcb_resource_init_root_field(&fcb_data);
1027         fcb_contract_init_root_field(&fcb_data);
1028
1029         fcb = forb_fres_contract_broker_new(orb, &impl, &fcb_data);
1030         if (!fcb) error(1, errno, "forb_fres_contract_broker_new failed");
1031
1032         /* Prepare executor before we register the fcb reference,
1033          * so that no reuqests are lost */
1034         ret = forb_executor_init(&executor);
1035         if (ret) error(1, errno, "forb_executor_init failed");
1036
1037         ret = forb_executor_register_object(&executor, fcb);
1038         if (ret) error(1, errno, "forb_executor_register_object failed");
1039
1040         ret = forb_register_reference(fcb, fres_contract_broker_reg_name);
1041         if (ret) error(1, errno, "forb_register_reference() failed");
1042
1043         ul_logmsg("Waiting for requests\n");
1044         ret = forb_executor_run(&executor);
1045         if (ret) error(1, errno, "forb_executor_run failed");
1046
1047         return 0;
1048 }