3 #include <sys/socket.h>
4 #include <netinet/in.h>
19 #include <semaphore.h>
21 #define MAX_STREAMS 10
22 #define MIN_GRANULARITY 100
24 unsigned opt_packet_size = 800;
25 int opt_send_buf_size = -1;
26 unsigned opt_period_usec = 10*MSEC_TO_USEC;
27 unsigned opt_jitter = 0;
28 char *opt_output = "delay_stats";
29 unsigned opt_count_sec = 0;
30 unsigned opt_def_bandwidth = 200;
31 unsigned opt_def_period_msec = 0;
32 int opt_granularity_usec = MIN_GRANULARITY;
33 bool opt_wait_for_queue_is_full; /* Don't gather any statistics until any queue is full */
35 bool some_queue_is_full = false;
36 struct timespec reset_timestamp;
38 /* Locked when some queue is full to prevent multiple resets of
40 pthread_mutex_t queue_full_mutex = PTHREAD_MUTEX_INITIALIZER;
42 int ac_sockfd[AC_NUM];
46 unsigned received, last_received;
53 /* maximal traffic delay in ms - 10 s*/
54 #define MAX_DELAY_US 10000000
56 unsigned delay_stats[AC_NUM][MAX_DELAY_US/MIN_GRANULARITY];
57 pthread_mutex_t delay_stats_mutex = PTHREAD_MUTEX_INITIALIZER;
59 /*struct ac_stats[AC_NUM] {
60 unsigned long int min_trans_time;
61 unsigned long int sum_trans_time;
62 struct timespec recv_timestamp;
63 struct timespec send_timestamp;
67 /* Input parameters */
69 int bandwidth_bps; /* bits per second */
70 int jitter; /* percent */
71 /* Mulualy exclusive input parameters */
73 long period_usec; /* all time units are in microseconds */
74 struct sockaddr_in rem_addr;
77 pthread_mutex_t mutex;
78 unsigned long long sent, really_sent, received;
82 struct send_endpoint sepoint[] = {
83 { .ac = AC_VO, .period_usec=200*MSEC_TO_USEC, .bandwidth_bps = 34*Kbit },
84 { .ac = AC_VI, .period_usec=25*MSEC_TO_USEC, .bandwidth_bps = 480*Kbit },
85 { .ac = AC_BE, .period_usec=40*MSEC_TO_USEC, .bandwidth_bps = 300*Kbit },
86 { .ac = AC_BK, .period_usec=40*MSEC_TO_USEC, .bandwidth_bps = 300*Kbit },
87 // { .ac = AC_VI, .period_usec=17*MSEC_TO_USEC, .bandwidth_bps = 675*Kbit },
91 struct stream streams[MAX_STREAMS];
93 unsigned int nr_streams = 0;
95 sem_t sem_thread_finished;
97 bool exit_flag = false;
104 /* Interrupt all receivers */
105 for (i=0; i < AC_NUM; i++) {
106 pthread_kill(receivers[i].thread, SIGUSR1);
110 void stream_to_text(char *stream_desc, size_t n, struct stream *stream, long long useconds)
116 snprintf(real, sizeof(real), "; real: %s sent %lld (%lld/s), received %lld (%lld/s)",
117 bandwidth_to_text(buf[0], (long long)stream->really_sent*stream->packet_size*8*SEC_TO_USEC/useconds),
118 stream->sent, stream->sent*SEC_TO_USEC/useconds,
119 stream->received, stream->received*SEC_TO_USEC/useconds);
124 snprintf(stream_desc, n, "%d: %s %s (%d bytes per %s +-%s, %d packets/s)%s",
125 stream-streams, ac_to_text[stream->ac], bandwidth_to_text(buf[0], stream->bandwidth_bps),
126 stream->packet_size, usec_to_text(buf[1], stream->period_usec),
127 usec_to_text(buf[2], stream->jitter*stream->period_usec/100),
128 (int)(SEC_TO_USEC/stream->period_usec), real);
131 void save_results(int argc, char *argv[], int useconds)
134 const int mini = 3000/opt_granularity_usec;
136 unsigned sum[AC_NUM];
139 fprintf(stderr, "Writing data to %s... ", logfname);
142 fprintf(logfd, "# Invoked as: ");
143 for (i=0; i<argc; i++) fprintf(logfd, "%s ", argv[i]);
144 fprintf(logfd, "\n");
146 if (useconds/SEC_TO_USEC != opt_count_sec) {
148 usec_to_text(buf, useconds);
149 fprintf(logfd, "# Data gathered for %s.\n", buf);
152 for (i = 0; i < nr_streams; i++) {
153 char stream_desc[200];
154 stream_to_text(stream_desc, sizeof(stream_desc), &streams[i], useconds);
155 fprintf(logfd, "# Stream %s\n", stream_desc);
158 /* Find maximal delay */
160 for (maxi = MAX_DELAY_US/opt_granularity_usec - 1; maxi >= 0; maxi--) {
161 for (ac = 0; ac < AC_NUM; ac++) {
162 if (delay_stats[ac][maxi] != 0) allzeros = false;
164 if (!allzeros) break;
167 if (maxi < mini) maxi = mini;
169 /* Calculate total number of sent packets per AC */
170 for (ac = 0; ac < AC_NUM; ac++) sum[ac] = 0;
171 for (i = 0; i < nr_streams; i++) {
173 sum[ac] += streams[i].sent;
178 for ( i = 0 ; i < maxi; i++) {
179 fprintf(logfd,"\n%f", i*opt_granularity_usec/1000.0);
180 for (ac = 0; ac < AC_NUM; ac++) {
182 val = (double)delay_stats[ac][i]*100.0 / sum[ac];
183 else val = -1; /* Don't display this ac */
184 fprintf(logfd," %lf", val);
188 fprintf(logfd,"\n\n");
192 for (ac = 0; ac < AC_NUM; ac++) {
193 unsigned long long integral = 0, last = -1;
195 fprintf(logfd,"%f %f\n", 0.0, 0.0);
199 while (delay_stats[ac][i] == 0) i++;
201 for (i++; i < maxi+1; i++) {
202 if (last != integral) {
203 val = (double)integral*100.0 / sum[ac];
204 fprintf(logfd,"%f %f\n", i*opt_granularity_usec/1000.0, val);
208 integral += delay_stats[ac][i-1];
211 fprintf(logfd,"\n\n");
214 fprintf(stderr, "finished.\n");
221 void timespec_add (struct timespec *sum, const struct timespec *left,
222 const struct timespec *right)
224 sum->tv_sec = left->tv_sec + right->tv_sec;
225 sum->tv_nsec = left->tv_nsec + right->tv_nsec;
227 if (sum->tv_nsec >= 1000000000){
229 sum->tv_nsec -= 1000000000;
234 void timespec_sub (struct timespec *diff, const struct timespec *left,
235 const struct timespec *right)
237 diff->tv_sec = left->tv_sec - right->tv_sec;
238 diff->tv_nsec = left->tv_nsec - right->tv_nsec;
240 if (diff->tv_nsec < 0){
242 diff->tv_nsec += 1000000000;
246 static inline long long timespec_sub_usec(const struct timespec *left,
247 const struct timespec *right)
249 struct timespec result;
250 timespec_sub(&result, left, right);
251 return (long long)result.tv_sec * SEC_TO_USEC +
252 result.tv_nsec / USEC_TO_NSEC;
255 static inline long long timespec2usec(const struct timespec *ts)
257 return ts->tv_sec * SEC_TO_USEC + ts->tv_nsec / USEC_TO_NSEC;
261 int create_ac_socket(unsigned int ac)
264 unsigned int yes=1, tos;
267 if ((sockfd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0)
269 perror("Unable to open socket");
272 if (fcntl(sockfd, F_SETFL, O_NONBLOCK) != 0) {
273 perror("set non-blocking socket");
276 if (setsockopt(sockfd,SOL_SOCKET,SO_REUSEADDR,&yes,sizeof(int)) == -1) {
277 perror("Unable to set socket");
281 if (opt_send_buf_size >= 0) {
282 if (setsockopt(sockfd,SOL_SOCKET,SO_SNDBUF,&opt_send_buf_size,sizeof(opt_send_buf_size)) == -1) {
283 perror("Unable to set socket buffer size");
289 //tos = ((AC_NUM - ac) *2 - 1)*32;
291 if (setsockopt(sockfd, SOL_IP, IP_TOS, &tos, sizeof(tos))) {
292 perror("Unable to set TOS");
304 void reset_statistics()
307 for (i = 0; i < nr_streams; i++) {
308 pthread_mutex_lock(&streams[i].mutex);
310 streams[i].really_sent = 0;
311 streams[i].received = 0;
312 pthread_mutex_unlock(&streams[i].mutex);
314 pthread_mutex_lock(&delay_stats_mutex);
315 clock_gettime(CLOCK_MONOTONIC, &reset_timestamp);
316 memset(delay_stats, 0, sizeof(delay_stats));
317 pthread_mutex_unlock(&delay_stats_mutex);
320 void* receiver(void* queue)
323 struct sockaddr_in rem_addr;
325 unsigned int ac, rem_addr_length;
326 long long int trans_time_usec, client_to_server_usec, server_to_client_usec;
327 long long int min_trans_time;
328 struct timespec send_timestamp, server_timestamp, recv_timestamp;
337 rem_addr_length = sizeof(rem_addr);
340 FD_SET(ac_sockfd[ac], &fdset);
341 ret = select(ac_sockfd[ac]+1, &fdset, NULL, NULL, NULL);
343 if (errno == EINTR) continue;
344 perror("receiver select");
347 mlen = recvfrom(ac_sockfd[ac], &msg, sizeof(msg), 0,
348 (struct sockaddr*)&rem_addr, &rem_addr_length);
350 perror("Chyba pri prijimani pozadavku");
353 clock_gettime(CLOCK_MONOTONIC,&recv_timestamp);
354 send_timestamp = msg.send_timestamp;
355 /* server_timestamp = msg.sendback_timestamp; */
357 /* Check whether this message was sent after reset_statistics() */
359 if ((ret = timespec_sub_usec(&send_timestamp, &reset_timestamp)) < 0) {
360 continue; /* If so, don't count it */
363 trans_time_usec = timespec_sub_usec(&recv_timestamp ,&send_timestamp);
364 client_to_server_usec = timespec_sub_usec(&server_timestamp, &send_timestamp);
365 server_to_client_usec = timespec_sub_usec(&recv_timestamp, &server_timestamp);
367 trans_time_usec /= 2;
369 if (trans_time_usec < MAX_DELAY_US) {
370 pthread_mutex_lock(&delay_stats_mutex);
371 delay_stats[ac][trans_time_usec/opt_granularity_usec]++;
372 pthread_mutex_unlock(&delay_stats_mutex);
374 receivers[ac].received++;
375 pthread_mutex_lock(&streams[msg.stream].mutex);
376 streams[msg.stream].received++;
377 pthread_mutex_unlock(&streams[msg.stream].mutex);
379 /*if (trans_time_nsec < min_trans_time)
380 min_trans_time = trans_time_nsec;*/
381 /*printf("seqn= %lu tos= %d start= %lu(s).%lu(ns)"\
382 "stop= %lu(s).%lu(ns)\n trans_time = %lums\n",\
383 msg.seqn, msg.tos, send_timestamp.tv_sec,\
384 send_timestamp.tv_nsec,recv_timestamp.tv_sec,\
385 recv_timestamp.tv_nsec, trans_time_msec); */
388 sem_post(&sem_thread_finished);
395 * @return -1 in case of error, 1 in case of sucessfull send and 0
396 * when all buffers are full.
399 send_packet(struct stream* stream, union msg_buff* buff)
402 while (sendto(ac_sockfd[stream->ac], buff, stream->packet_size, 0,
403 (struct sockaddr*)&stream->rem_addr, sizeof(stream->rem_addr)) < 0) {
404 if (errno == EINTR) continue;
405 if (errno == EAGAIN) {
406 if (opt_wait_for_queue_is_full &&
407 !some_queue_is_full &&
408 /* We use mutex as atomic test and set */
409 (pthread_mutex_trylock(&queue_full_mutex) != EBUSY)) {
410 some_queue_is_full = true;
416 perror("Error while sending");
425 wait_for_next_send(struct stream* stream, struct timespec *last_send_time)
427 struct timespec time_to_wait, current_time, period, interval;
428 unsigned period_usec = stream->period_usec;
430 /* |~~~+~~~| jitter interval (width = 2*stream->jitter percentage from period)*/
431 /* |-------------| nominal period*/
432 if (stream->jitter) {
433 period.tv_nsec = USEC_TO_NSEC*(period_usec*(100-stream->jitter)/100
434 + rand() % (2*period_usec*stream->jitter/100));
436 period.tv_nsec = USEC_TO_NSEC*(period_usec);
440 timespec_add(&time_to_wait, last_send_time, &period);
441 clock_gettime(CLOCK_MONOTONIC,¤t_time);
442 timespec_sub(&interval,&time_to_wait,¤t_time);
443 nanosleep(&interval,NULL);
447 void* sender(void* arg)
450 unsigned long int seqn;
451 struct stream* stream = (struct stream*) arg;
452 char stream_desc[100];
455 stream_to_text(stream_desc, sizeof(stream_desc), stream, 0);
456 printf("%s\n", stream_desc);
458 if (stream->bandwidth_bps == 0)
464 set_rt_prio(90-stream->ac);
468 /* buff.msg.seqn = seqn++; */
469 /* buff.msg.tos = ac_to_tos[stream->ac]; */
470 buff.msg.stream = stream-streams;
472 clock_gettime(CLOCK_MONOTONIC,&buff.msg.send_timestamp);
474 ret = send_packet(stream, &buff);
479 pthread_mutex_lock(&stream->mutex);
482 stream->really_sent++;
483 pthread_mutex_unlock(&stream->mutex);
486 printf("%d", stream->ac);
490 wait_for_next_send(stream, &buff.msg.send_timestamp);
493 sem_post(&sem_thread_finished);
498 calc_stream_params(struct stream *stream)
501 unsigned period_usec;
505 bandwidth = stream->bandwidth_bps;
507 /* Avoid arithmetic exception. Server thread will exit if
508 stream->bandwidth_bps == 0. */
509 if (bandwidth == 0) bandwidth = 1;
511 if (stream->packet_size) {
512 packet_size = stream->packet_size;
513 period_usec = SEC_TO_USEC*packet_size*8/bandwidth;
514 if (period_usec == 0) period_usec = 1;
515 } else if (stream->period_usec) {
516 period_usec = stream->period_usec;
517 packet_size = (long long)bandwidth/8 * period_usec/SEC_TO_USEC;
520 stream_to_text(buf, sizeof(buf), stream, 0);
521 fprintf(stderr, "Neither packet size nor period was specified for a stream %s\n", buf);
525 if (packet_size < sizeof(struct msg_t)) {
526 fprintf(stderr, "Packet size too small (min %d)\n", sizeof(struct msg_t));
530 stream->packet_size = packet_size;
531 stream->period_usec = period_usec;
532 stream->jitter = opt_jitter;
534 memset(&stream->rem_addr,0, sizeof(stream->rem_addr));
536 stream->rem_addr.sin_family = AF_INET;
537 ph = gethostbyname(server_addr);
539 stream->rem_addr.sin_addr = *((struct in_addr *)ph->h_addr);
541 perror("Unknown server");
544 stream->rem_addr.sin_port = htons(BASE_PORT + stream->ac);
549 * Parse -b parameter.
551 * @param params String to parse
553 * @return NULL in case of success, pointer to a problematic character
556 char* parse_bandwidths(char *params)
558 struct stream *sp = &streams[nr_streams];
560 while (*params && nr_streams < MAX_STREAMS) {
561 char *ac_ids[AC_NUM] = { [AC_VO]="VO", [AC_VI]="VI", [AC_BE]="BE", [AC_BK]="BK" };
565 if (strlen(params) < 2)
567 for (i=0; i<AC_NUM; i++) {
568 if (strncmp(params, ac_ids[i], 2) == 0) {
570 params+=strlen(ac_ids[i]);
578 if (*params == ':') {
581 bw = strtol(params, &next_char, 10);
582 if (next_char == params)
586 bw = opt_def_bandwidth;
588 sp->bandwidth_bps = bw*Kbit;
591 long packet_size = 0;
592 if (*params == '@') {
594 period = strtol(params, &next_char, 10);
600 if (*params == '/') {
602 packet_size = strtol(params, &next_char, 10);
603 if (packet_size == 0)
607 packet_size = opt_packet_size;
608 period = opt_def_period_msec;
611 sp->period_usec = period*MSEC_TO_USEC;
612 sp->packet_size = packet_size;
616 if (*params != '\0' && *params != ',')
626 int main(int argc, char *argv[])
628 int ac, i, rc, seconds;
634 while ((opt = getopt(argc, argv, "B:b:c:g:j:o:qQ:s:T:")) != -1) {
637 opt_def_bandwidth = atoi(optarg);
641 error = parse_bandwidths(optarg);
644 fprintf(stderr, "Bandwidth parse error - string to short\n");
646 fprintf(stderr, "Bandwidth parse error at '%s'\n", error);
652 opt_count_sec = atoi(optarg);
655 opt_granularity_usec = atoi(optarg);
656 if (opt_granularity_usec < MIN_GRANULARITY) {
657 fprintf(stderr, "Granulatiry too small (min %d)!\n", MIN_GRANULARITY);
662 opt_jitter = atoi(optarg);
668 opt_send_buf_size = atoi(optarg);
671 opt_wait_for_queue_is_full = true;
674 opt_packet_size = atoi(optarg);
677 opt_def_period_msec = atoi(optarg);
680 fprintf(stderr, "Usage: %s [ options ] server_addr\n\n", argv[0]);
681 fprintf(stderr, "Options:\n");
682 fprintf(stderr, " -B default bandwidth for -b option [kbit]\n");
683 fprintf(stderr, " -b bandwidth of streams (VO|VI|BE|BK)[:<kbit>][@<msec> or /<bytes>][,...]\n");
684 fprintf(stderr, " -c count (number of seconds to run)\n");
685 fprintf(stderr, " -g histogram granularity [usec]\n");
686 fprintf(stderr, " -j send jitter (0-100) [%%]\n");
687 fprintf(stderr, " -o output filename (.dat will be appended)\n");
688 fprintf(stderr, " -q gather statistics only after some queue becomes full\n");
689 fprintf(stderr, " -Q <bytes> set size for socket send buffers\n");
690 fprintf(stderr, " -s size of data payload in packets [bytes] (default: %d)\n", opt_packet_size);
691 fprintf(stderr, " -T default period for -b option [msec]\n");
695 if (opt_packet_size && opt_def_period_msec) {
696 fprintf(stderr, "Error: Nonzero -T and -s can't be used together!.\n");
701 server_addr = argv[optind];
703 fprintf(stderr, "Expected server address argument\n");
708 parse_bandwidths("BE");
710 pthread_attr_init(&attr);
712 snprintf(logfname, sizeof(logfname), "%s.dat", opt_output);
714 if ((logfd = fopen(logfname,"w+")) == NULL) {
715 fprintf(stderr,"Can not open %s\n", logfname);
718 if (signal(SIGTERM, stopper) == SIG_ERR) {
719 perror("Error in signal registration");
723 if (signal(SIGINT, stopper) == SIG_ERR) {
724 perror("Signal handler registration error");
729 sa.sa_handler = empty_handler;
730 sa.sa_flags = 0; /* don't restart syscalls */
732 if (sigaction(SIGUSR1, &sa, NULL) < 0) {
733 perror("sigaction error");
737 sem_init(&sem_thread_finished, 0, 0);
741 /* create four receivers each per AC */
742 for (ac = AC_NUM - 1; ac >= 0; ac--) {
743 ac_sockfd[ac] = create_ac_socket(ac);
744 if (ac_sockfd[ac] < 0) {
747 rc = pthread_create(&receivers[ac].thread, &attr, receiver, (void*) ac);
749 fprintf(stderr, "Error while creating receiver %d\n",rc);
754 /* create sendpoints */
755 for (i = 0; i < nr_streams; i++) {
756 struct stream *s = &streams[i];
757 pthread_mutex_init(&s->mutex, NULL);
758 calc_stream_params(s);
759 rc = pthread_create(&thread, &attr, sender, (void*) s);
761 fprintf(stderr, "Error while creating sender %d\n",rc);
770 fprintf(stderr, "\r%3ds", seconds);
771 for (ac = 0; ac < AC_NUM; ac++) {
772 int delta = receivers[ac].received - receivers[ac].last_received;
773 receivers[ac].last_received = receivers[ac].received;
774 fprintf(stderr, " %s %5d %4d/s", ac_to_text[ac], receivers[ac].received, delta);
777 if (seconds == opt_count_sec)
781 fprintf(stderr, "\nWaiting for threads to finish\n");
782 /* Wait for all threads to finish */
783 for (i=0; i < nr_streams + AC_NUM; i++) {
784 sem_wait(&sem_thread_finished);
786 struct timespec end_timestamp, measure_length;
787 clock_gettime(CLOCK_MONOTONIC,&end_timestamp);
788 timespec_sub(&measure_length, &end_timestamp, &reset_timestamp);
790 save_results(argc, argv, timespec2usec(&measure_length));