2 * drivers/misc/tegra-profiler/comm.c
4 * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
20 #include <linux/slab.h>
21 #include <linux/vmalloc.h>
22 #include <linux/miscdevice.h>
23 #include <linux/sched.h>
24 #include <linux/poll.h>
25 #include <linux/bitops.h>
26 #include <linux/err.h>
28 #include <asm/uaccess.h>
30 #include <linux/tegra_profiler.h>
35 struct quadd_comm_ctx comm_ctx;
37 static inline void *rb_alloc(unsigned long size)
42 static inline void rb_free(void *addr)
47 static void rb_reset(struct quadd_ring_buffer *rb)
52 rb->max_fill_count = 0;
55 static int rb_init(struct quadd_ring_buffer *rb, size_t size)
57 spin_lock_init(&rb->lock);
62 rb->buf = (char *) rb_alloc(rb->size);
64 pr_err("Ring buffer alloc error\n");
67 pr_info("rb: data buffer size: %u\n", (unsigned int)rb->size);
74 static void rb_deinit(struct quadd_ring_buffer *rb)
78 spin_lock_irqsave(&rb->lock, flags);
85 spin_unlock_irqrestore(&rb->lock, flags);
88 static __attribute__((unused)) int rb_is_full(struct quadd_ring_buffer *rb)
90 return rb->fill_count == rb->size;
93 static int rb_is_empty(struct quadd_ring_buffer *rb)
95 return rb->fill_count == 0;
98 static int rb_is_empty_lock(struct quadd_ring_buffer *rb)
103 spin_lock_irqsave(&rb->lock, flags);
104 res = rb->fill_count == 0;
105 spin_unlock_irqrestore(&rb->lock, flags);
111 rb_get_free_space(struct quadd_ring_buffer *rb)
113 return rb->size - rb->fill_count;
117 rb_write(struct quadd_ring_buffer *rb, char *data, size_t length)
119 size_t new_pos_write, chunk1;
121 if (length > rb_get_free_space(rb))
124 new_pos_write = (rb->pos_write + length) % rb->size;
126 if (new_pos_write < rb->pos_write) {
127 chunk1 = rb->size - rb->pos_write;
128 memcpy(rb->buf + rb->pos_write, data, chunk1);
129 if (new_pos_write > 0)
130 memcpy(rb->buf, data + chunk1, new_pos_write);
132 memcpy(rb->buf + rb->pos_write, data, length);
135 rb->pos_write = new_pos_write;
136 rb->fill_count += length;
141 static ssize_t rb_read_undo(struct quadd_ring_buffer *rb, size_t length)
143 if (rb_get_free_space(rb) < length)
146 if (rb->pos_read > length)
147 rb->pos_read -= length;
149 rb->pos_read += rb->size - length;
151 rb->fill_count += length;
155 static size_t rb_read(struct quadd_ring_buffer *rb, char *data, size_t length)
157 unsigned int new_pos_read, chunk1;
159 if (length > rb->fill_count)
162 new_pos_read = (rb->pos_read + length) % rb->size;
164 if (new_pos_read < rb->pos_read) {
165 chunk1 = rb->size - rb->pos_read;
166 memcpy(data, rb->buf + rb->pos_read, chunk1);
167 if (new_pos_read > 0)
168 memcpy(data + chunk1, rb->buf, new_pos_read);
170 memcpy(data, rb->buf + rb->pos_read, length);
173 rb->pos_read = new_pos_read;
174 rb->fill_count -= length;
180 rb_read_user(struct quadd_ring_buffer *rb, char __user *data, size_t length)
182 size_t new_pos_read, chunk1;
184 if (length > rb->fill_count)
187 new_pos_read = (rb->pos_read + length) % rb->size;
189 if (new_pos_read < rb->pos_read) {
190 chunk1 = rb->size - rb->pos_read;
191 if (copy_to_user(data, rb->buf + rb->pos_read, chunk1))
194 if (new_pos_read > 0) {
195 if (copy_to_user(data + chunk1, rb->buf,
200 if (copy_to_user(data, rb->buf + rb->pos_read, length))
204 rb->pos_read = new_pos_read;
205 rb->fill_count -= length;
211 write_sample(struct quadd_record_data *sample,
212 struct quadd_iovec *vec, int vec_count)
216 struct quadd_ring_buffer *rb = &comm_ctx.rb;
217 size_t length_sample;
219 length_sample = sizeof(struct quadd_record_data);
220 for (i = 0; i < vec_count; i++)
221 length_sample += vec[i].len;
223 spin_lock_irqsave(&rb->lock, flags);
225 if (length_sample > rb_get_free_space(rb)) {
226 pr_err_once("Error: Buffer has been overflowed\n");
227 spin_unlock_irqrestore(&rb->lock, flags);
231 if (!rb_write(rb, (char *)sample, sizeof(struct quadd_record_data))) {
232 spin_unlock_irqrestore(&rb->lock, flags);
236 for (i = 0; i < vec_count; i++) {
237 if (!rb_write(rb, vec[i].base, vec[i].len)) {
238 spin_unlock_irqrestore(&rb->lock, flags);
239 pr_err_once("%s: error: ring buffer\n", __func__);
244 if (rb->fill_count > rb->max_fill_count)
245 rb->max_fill_count = rb->fill_count;
247 spin_unlock_irqrestore(&rb->lock, flags);
249 wake_up_interruptible(&comm_ctx.read_wait);
252 static ssize_t read_sample(char __user *buffer, size_t max_length)
256 int retval = -EIO, ip_size;
257 int was_read = 0, write_offset = 0;
259 struct quadd_ring_buffer *rb = &comm_ctx.rb;
260 struct quadd_record_data record;
261 size_t length_extra = 0, nr_events;
262 struct quadd_sample_data *sample;
264 spin_lock_irqsave(&rb->lock, flags);
266 if (rb_is_empty(rb)) {
271 if (rb->fill_count < sizeof(record))
274 if (!rb_read(rb, (char *)&record, sizeof(record)))
277 was_read += sizeof(record);
279 type = record.record_type;
282 case QUADD_RECORD_TYPE_SAMPLE:
283 sample = &record.sample;
285 if (rb->fill_count < sizeof(sed))
288 if (!rb_read(rb, (char *)&sed, sizeof(sed)))
291 was_read += sizeof(sed);
293 ip_size = (sed & QUADD_SED_IP64) ?
294 sizeof(u64) : sizeof(u32);
296 length_extra = sample->callchain_nr * ip_size;
298 nr_events = __sw_hweight32(sample->events_flags);
299 length_extra += nr_events * sizeof(u32);
301 length_extra += sample->state ? sizeof(u32) : 0;
304 case QUADD_RECORD_TYPE_MMAP:
305 length_extra = sizeof(u64) * 2;
307 if (record.mmap.filename_length > 0) {
308 length_extra += record.mmap.filename_length;
310 pr_err("Error: filename is empty\n");
315 case QUADD_RECORD_TYPE_HEADER:
316 length_extra = record.hdr.nr_events * sizeof(u32);
319 case QUADD_RECORD_TYPE_DEBUG:
320 length_extra = record.debug.extra_length;
323 case QUADD_RECORD_TYPE_MA:
327 case QUADD_RECORD_TYPE_POWER_RATE:
328 length_extra = record.power_rate.nr_cpus * sizeof(u32);
331 case QUADD_RECORD_TYPE_ADDITIONAL_SAMPLE:
332 length_extra = record.additional_sample.extra_length;
339 if (was_read + length_extra > max_length) {
340 retval = rb_read_undo(rb, was_read);
348 if (length_extra > rb->fill_count)
351 if (copy_to_user(buffer, &record, sizeof(record)))
352 goto out_fault_error;
354 write_offset += sizeof(record);
356 if (type == QUADD_RECORD_TYPE_SAMPLE) {
357 if (copy_to_user(buffer + write_offset, &sed, sizeof(sed)))
358 goto out_fault_error;
360 write_offset += sizeof(sed);
363 if (length_extra > 0) {
364 retval = rb_read_user(rb, buffer + write_offset,
369 write_offset += length_extra;
372 spin_unlock_irqrestore(&rb->lock, flags);
379 spin_unlock_irqrestore(&rb->lock, flags);
383 static void put_sample(struct quadd_record_data *data,
384 struct quadd_iovec *vec, int vec_count)
386 if (!atomic_read(&comm_ctx.active))
389 write_sample(data, vec, vec_count);
392 static void comm_reset(void)
396 pr_debug("Comm reset\n");
397 spin_lock_irqsave(&comm_ctx.rb.lock, flags);
398 rb_reset(&comm_ctx.rb);
399 spin_unlock_irqrestore(&comm_ctx.rb.lock, flags);
402 static int is_active(void)
404 return atomic_read(&comm_ctx.active) != 0;
407 static struct quadd_comm_data_interface comm_data = {
408 .put_sample = put_sample,
410 .is_active = is_active,
413 static int check_access_permission(void)
415 struct task_struct *task;
417 if (capable(CAP_SYS_ADMIN))
420 if (!comm_ctx.params_ok || comm_ctx.process_pid == 0)
424 task = pid_task(find_vpid(comm_ctx.process_pid), PIDTYPE_PID);
429 if (current_fsuid() != task_uid(task) &&
430 task_uid(task) != comm_ctx.debug_app_uid) {
431 pr_err("Permission denied, owner/task uids: %u/%u\n",
432 current_fsuid(), task_uid(task));
438 static int device_open(struct inode *inode, struct file *file)
440 mutex_lock(&comm_ctx.io_mutex);
442 mutex_unlock(&comm_ctx.io_mutex);
446 static int device_release(struct inode *inode, struct file *file)
448 mutex_lock(&comm_ctx.io_mutex);
451 if (comm_ctx.nr_users == 0) {
452 if (atomic_cmpxchg(&comm_ctx.active, 1, 0)) {
453 comm_ctx.control->stop();
454 pr_info("Stop profiling: daemon is closed\n");
457 mutex_unlock(&comm_ctx.io_mutex);
463 device_poll(struct file *file, poll_table *wait)
465 unsigned int mask = 0;
466 struct quadd_ring_buffer *rb = &comm_ctx.rb;
468 poll_wait(file, &comm_ctx.read_wait, wait);
470 if (!rb_is_empty_lock(rb))
471 mask |= POLLIN | POLLRDNORM;
473 if (!atomic_read(&comm_ctx.active))
480 device_read(struct file *filp,
487 size_t samples_counter = 0;
488 size_t was_read = 0, min_size;
490 err = check_access_permission();
494 mutex_lock(&comm_ctx.io_mutex);
496 if (!atomic_read(&comm_ctx.active)) {
497 mutex_unlock(&comm_ctx.io_mutex);
501 min_size = sizeof(struct quadd_record_data) + sizeof(u32);
503 while (was_read + min_size < length) {
504 res = read_sample(buffer + was_read, length - was_read);
506 mutex_unlock(&comm_ctx.io_mutex);
507 pr_err("Error: data is corrupted\n");
517 if (!atomic_read(&comm_ctx.active))
521 mutex_unlock(&comm_ctx.io_mutex);
526 device_ioctl(struct file *file,
527 unsigned int ioctl_num,
528 unsigned long ioctl_param)
531 struct quadd_parameters *user_params;
532 struct quadd_comm_cap cap;
533 struct quadd_module_state state;
534 struct quadd_module_version versions;
535 struct quadd_extables extabs;
537 struct quadd_ring_buffer *rb = &comm_ctx.rb;
539 if (ioctl_num != IOCTL_SETUP &&
540 ioctl_num != IOCTL_GET_CAP &&
541 ioctl_num != IOCTL_GET_STATE &&
542 ioctl_num != IOCTL_GET_VERSION) {
543 err = check_access_permission();
548 mutex_lock(&comm_ctx.io_mutex);
552 if (atomic_read(&comm_ctx.active)) {
553 pr_err("error: tegra profiler is active\n");
558 user_params = vmalloc(sizeof(*user_params));
564 if (copy_from_user(user_params, (void __user *)ioctl_param,
565 sizeof(struct quadd_parameters))) {
566 pr_err("setup failed\n");
572 err = comm_ctx.control->set_parameters(user_params,
573 &comm_ctx.debug_app_uid);
575 pr_err("error: setup failed\n");
579 comm_ctx.params_ok = 1;
580 comm_ctx.process_pid = user_params->pids[0];
582 if (user_params->reserved[QUADD_PARAM_IDX_SIZE_OF_RB] == 0) {
583 pr_err("error: too old version of daemon\n");
588 comm_ctx.rb_size = user_params->reserved[0];
590 pr_info("setup success: freq/mafreq: %u/%u, backtrace: %d, pid: %d\n",
592 user_params->ma_freq,
593 user_params->backtrace,
594 user_params->pids[0]);
600 comm_ctx.control->get_capabilities(&cap);
601 if (copy_to_user((void __user *)ioctl_param, &cap,
602 sizeof(struct quadd_comm_cap))) {
603 pr_err("error: get_capabilities failed\n");
609 case IOCTL_GET_VERSION:
610 strcpy((char *)versions.branch, QUADD_MODULE_BRANCH);
611 strcpy((char *)versions.version, QUADD_MODULE_VERSION);
613 versions.samples_version = QUADD_SAMPLES_VERSION;
614 versions.io_version = QUADD_IO_VERSION;
616 if (copy_to_user((void __user *)ioctl_param, &versions,
617 sizeof(struct quadd_module_version))) {
618 pr_err("error: get version failed\n");
624 case IOCTL_GET_STATE:
625 comm_ctx.control->get_state(&state);
627 state.buffer_size = comm_ctx.rb_size;
629 spin_lock_irqsave(&rb->lock, flags);
630 state.buffer_fill_size =
631 comm_ctx.rb_size - rb_get_free_space(rb);
632 state.reserved[QUADD_MOD_STATE_IDX_RB_MAX_FILL_COUNT] =
634 spin_unlock_irqrestore(&rb->lock, flags);
636 if (copy_to_user((void __user *)ioctl_param, &state,
637 sizeof(struct quadd_module_state))) {
638 pr_err("error: get_state failed\n");
645 if (!atomic_cmpxchg(&comm_ctx.active, 0, 1)) {
646 if (!comm_ctx.params_ok) {
647 pr_err("error: params failed\n");
648 atomic_set(&comm_ctx.active, 0);
653 err = rb_init(rb, comm_ctx.rb_size);
655 pr_err("error: rb_init failed\n");
656 atomic_set(&comm_ctx.active, 0);
660 err = comm_ctx.control->start();
662 pr_err("error: start failed\n");
663 atomic_set(&comm_ctx.active, 0);
666 pr_info("Start profiling success\n");
671 if (atomic_cmpxchg(&comm_ctx.active, 1, 0)) {
672 comm_ctx.control->stop();
673 wake_up_interruptible(&comm_ctx.read_wait);
674 rb_deinit(&comm_ctx.rb);
675 pr_info("Stop profiling success\n");
679 case IOCTL_SET_EXTAB:
680 if (copy_from_user(&extabs, (void __user *)ioctl_param,
682 pr_err("error: set_extab failed\n");
687 err = comm_ctx.control->set_extab(&extabs);
689 pr_err("error: set_extab\n");
695 pr_err("error: ioctl %u is unsupported in this version of module\n",
701 mutex_unlock(&comm_ctx.io_mutex);
705 static void unregister(void)
707 misc_deregister(comm_ctx.misc_dev);
708 kfree(comm_ctx.misc_dev);
711 static void free_ctx(void)
713 rb_deinit(&comm_ctx.rb);
716 static const struct file_operations qm_fops = {
720 .release = device_release,
721 .unlocked_ioctl = device_ioctl,
722 .compat_ioctl = device_ioctl,
725 static int comm_init(void)
728 struct miscdevice *misc_dev;
730 misc_dev = kzalloc(sizeof(*misc_dev), GFP_KERNEL);
732 pr_err("Error: alloc error\n");
736 misc_dev->minor = MISC_DYNAMIC_MINOR;
737 misc_dev->name = QUADD_DEVICE_NAME;
738 misc_dev->fops = &qm_fops;
740 res = misc_register(misc_dev);
742 pr_err("Error: misc_register: %d\n", res);
745 comm_ctx.misc_dev = misc_dev;
747 mutex_init(&comm_ctx.io_mutex);
748 atomic_set(&comm_ctx.active, 0);
750 comm_ctx.params_ok = 0;
751 comm_ctx.process_pid = 0;
752 comm_ctx.nr_users = 0;
754 init_waitqueue_head(&comm_ctx.read_wait);
759 struct quadd_comm_data_interface *
760 quadd_comm_events_init(struct quadd_comm_control_interface *control)
768 comm_ctx.control = control;
772 void quadd_comm_events_exit(void)
774 mutex_lock(&comm_ctx.io_mutex);
777 mutex_unlock(&comm_ctx.io_mutex);