drivers/misc/tegra-profiler/backtrace.c

   1 /*
   2  * drivers/misc/tegra-profiler/backtrace.c
   3  *
   4  * Copyright (c) 2015, NVIDIA CORPORATION.  All rights reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  */
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/uaccess.h>
  20 #include <linux/sched.h>
  21 #include <linux/mm.h>
  22
  23 #include <linux/tegra_profiler.h>
  24
  25 #include "quadd.h"
  26 #include "backtrace.h"
  27 #include "eh_unwind.h"
  28 #include "dwarf_unwind.h"
  29 #include "hrt.h"
  30 #include "tegra.h"
  31
  32 unsigned long
  33 quadd_user_stack_pointer(struct pt_regs *regs)
  34 {
  35 #ifdef CONFIG_ARM64
  36         if (compat_user_mode(regs))
  37                 return regs->compat_sp;
  38 #endif
  39
  40         return user_stack_pointer(regs);
  41 }
  42
  43 unsigned long
  44 quadd_get_user_frame_pointer(struct pt_regs *regs)
  45 {
  46         unsigned long fp;
  47
  48 #ifdef CONFIG_ARM64
  49         if (compat_user_mode(regs))
  50                 fp = is_thumb_mode(regs) ?
  51                         regs->compat_usr(7) : regs->compat_usr(11);
  52         else
  53                 fp = regs->regs[29];
  54 #else
  55         fp = is_thumb_mode(regs) ? regs->ARM_r7 : regs->ARM_fp;
  56 #endif
  57         return fp;
  58 }
  59
  60 unsigned long
  61 quadd_user_link_register(struct pt_regs *regs)
  62 {
  63 #ifdef CONFIG_ARM64
  64         return compat_user_mode(regs) ?
  65                 regs->compat_lr : regs->regs[30];
  66 #else
  67         return regs->ARM_lr;
  68 #endif
  69 }
  70
  71 static inline void
  72 put_unw_type(u32 *p, int bt_idx, unsigned int type)
  73 {
  74         int word_idx, shift;
  75
  76         word_idx = bt_idx / 8;
  77         shift = (bt_idx % 8) * 4;
  78
  79         *(p + word_idx) &= ~(0x0f << shift);
  80         *(p + word_idx) |= (type & 0x0f) << shift;
  81 }
  82
  83 int
  84 quadd_callchain_store(struct quadd_callchain *cc,
  85                       unsigned long ip, unsigned int type)
  86 {
  87         unsigned long low_addr = cc->hrt->low_addr;
  88
  89         if (ip < low_addr || !validate_pc_addr(ip, sizeof(unsigned long))) {
  90                 cc->unw_rc = QUADD_URC_PC_INCORRECT;
  91                 return 0;
  92         }
  93
  94         if (cc->nr >= QUADD_MAX_STACK_DEPTH) {
  95                 cc->unw_rc = QUADD_URC_LEVEL_TOO_DEEP;
  96                 return 0;
  97         }
  98
  99         put_unw_type(cc->types, cc->nr, type);
 100
 101         if (cc->cs_64)
 102                 cc->ip_64[cc->nr++] = ip;
 103         else
 104                 cc->ip_32[cc->nr++] = ip;
 105
 106         return 1;
 107 }
 108
 109 static int
 110 is_ex_entry_exist(struct pt_regs *regs,
 111                   unsigned long addr,
 112                   struct task_struct *task)
 113 {
 114         return quadd_is_ex_entry_exist_dwarf(regs, addr, task) ||
 115                quadd_is_ex_entry_exist_arm32_ehabi(regs, addr, task);
 116 }
 117
 118 static unsigned long __user *
 119 user_backtrace(struct pt_regs *regs,
 120                unsigned long __user *tail,
 121                struct quadd_callchain *cc,
 122                struct vm_area_struct *stack_vma,
 123                struct task_struct *task)
 124 {
 125         int nr_added;
 126         unsigned long value, value_lr = 0, value_fp = 0;
 127         unsigned long __user *fp_prev = NULL;
 128
 129         if (!is_vma_addr((unsigned long)tail, stack_vma, sizeof(*tail)))
 130                 return NULL;
 131
 132         if (__copy_from_user_inatomic(&value, tail, sizeof(unsigned long))) {
 133                 cc->unw_rc = QUADD_URC_EACCESS;
 134                 return NULL;
 135         }
 136
 137         if (is_vma_addr(value, stack_vma, sizeof(value))) {
 138                 /* gcc thumb/clang frame */
 139                 value_fp = value;
 140
 141                 if (!is_vma_addr((unsigned long)(tail + 1), stack_vma,
 142                     sizeof(*tail)))
 143                         return NULL;
 144
 145                 if (__copy_from_user_inatomic(&value_lr, tail + 1,
 146                                               sizeof(value_lr))) {
 147                         cc->unw_rc = QUADD_URC_EACCESS;
 148                         return NULL;
 149                 }
 150
 151                 cc->curr_fp = value_fp;
 152                 cc->curr_sp = (unsigned long)tail + sizeof(value_fp) * 2;
 153                 cc->curr_pc = value_lr;
 154         } else {
 155                 /* gcc arm frame */
 156                 if (__copy_from_user_inatomic(&value_fp, tail - 1,
 157                                               sizeof(value_fp))) {
 158                         cc->unw_rc = QUADD_URC_EACCESS;
 159                         return NULL;
 160                 }
 161
 162                 if (!is_vma_addr(value_fp, stack_vma, sizeof(value_fp)))
 163                         return NULL;
 164
 165                 cc->curr_fp = value_fp;
 166                 cc->curr_sp = (unsigned long)tail + sizeof(value_fp);
 167                 cc->curr_pc = value_lr = value;
 168         }
 169
 170         fp_prev = (unsigned long __user *)value_fp;
 171         if (fp_prev <= tail)
 172                 return NULL;
 173
 174         nr_added = quadd_callchain_store(cc, value_lr, QUADD_UNW_TYPE_FP);
 175         if (nr_added == 0)
 176                 return NULL;
 177
 178         if (cc->unw_method == QUADD_UNW_METHOD_MIXED &&
 179             is_ex_entry_exist(regs, value_lr, task))
 180                 return NULL;
 181
 182         return fp_prev;
 183 }
 184
 185 static unsigned int
 186 get_user_callchain_fp(struct pt_regs *regs,
 187                       struct quadd_callchain *cc,
 188                       struct task_struct *task)
 189 {
 190         unsigned long fp, sp, pc, reg;
 191         struct vm_area_struct *vma, *vma_pc = NULL;
 192         unsigned long __user *tail = NULL;
 193         struct mm_struct *mm = task->mm;
 194
 195         cc->nr = 0;
 196         cc->unw_rc = QUADD_URC_FP_INCORRECT;
 197
 198         if (!regs || !mm) {
 199                 cc->unw_rc = QUADD_URC_FAILURE;
 200                 return 0;
 201         }
 202
 203         sp = quadd_user_stack_pointer(regs);
 204         pc = instruction_pointer(regs);
 205         fp = quadd_get_user_frame_pointer(regs);
 206
 207         if (fp == 0 || fp < sp || fp & 0x3)
 208                 return 0;
 209
 210         vma = find_vma(mm, sp);
 211         if (!vma) {
 212                 cc->unw_rc = QUADD_URC_SP_INCORRECT;
 213                 return 0;
 214         }
 215
 216         if (!is_vma_addr(fp, vma, sizeof(fp)))
 217                 return 0;
 218
 219         if (probe_kernel_address(fp, reg)) {
 220                 pr_warn_once("%s: failed for address: %#lx\n", __func__, fp);
 221                 cc->unw_rc = QUADD_URC_EACCESS;
 222                 return 0;
 223         }
 224
 225         pr_debug("sp/fp: %#lx/%#lx, pc/lr: %#lx/%#lx, *fp: %#lx, stack: %#lx-%#lx\n",
 226                  sp, fp, pc, quadd_user_link_register(regs), reg,
 227                  vma->vm_start, vma->vm_end);
 228
 229         if (is_thumb_mode(regs)) {
 230                 if (reg <= fp || !is_vma_addr(reg, vma, sizeof(reg)))
 231                         return 0;
 232         } else if (reg > fp && is_vma_addr(reg, vma, sizeof(reg))) {
 233                 /* fp --> fp prev */
 234                 unsigned long value;
 235                 int read_lr = 0;
 236
 237                 if (is_vma_addr(fp + sizeof(unsigned long), vma, sizeof(fp))) {
 238                         if (__copy_from_user_inatomic(
 239                                         &value,
 240                                         (unsigned long __user *)fp + 1,
 241                                         sizeof(unsigned long))) {
 242                                 cc->unw_rc = QUADD_URC_EACCESS;
 243                                 return 0;
 244                         }
 245
 246                         vma_pc = find_vma(mm, pc);
 247                         read_lr = 1;
 248                 }
 249
 250                 if (!read_lr || !is_vma_addr(value, vma_pc, sizeof(value))) {
 251                         /* gcc: fp --> short frame tail (fp) */
 252                         int nr_added;
 253                         unsigned long lr = quadd_user_link_register(regs);
 254
 255                         nr_added = quadd_callchain_store(cc, lr,
 256                                                          QUADD_UNW_TYPE_FP);
 257                         if (nr_added == 0)
 258                                 return cc->nr;
 259
 260                         tail = (unsigned long __user *)reg;
 261                 }
 262         }
 263
 264         if (!tail)
 265                 tail = (unsigned long __user *)fp;
 266
 267         while (tail && !((unsigned long)tail & 0x3))
 268                 tail = user_backtrace(regs, tail, cc, vma, task);
 269
 270         return cc->nr;
 271 }
 272
 273 static unsigned int
 274 __user_backtrace(struct pt_regs *regs,
 275                  struct quadd_callchain *cc,
 276                  struct task_struct *task)
 277 {
 278         struct mm_struct *mm = task->mm;
 279         struct vm_area_struct *vma;
 280         unsigned long __user *tail;
 281
 282         cc->unw_rc = QUADD_URC_FP_INCORRECT;
 283
 284         if (!mm) {
 285                 cc->unw_rc = QUADD_URC_FAILURE;
 286                 return cc->nr;
 287         }
 288
 289         vma = find_vma(mm, cc->curr_sp);
 290         if (!vma) {
 291                 cc->unw_rc = QUADD_URC_SP_INCORRECT;
 292                 return cc->nr;
 293         }
 294
 295         tail = (unsigned long __user *)cc->curr_fp;
 296
 297         while (tail && !((unsigned long)tail & 0x3))
 298                 tail = user_backtrace(regs, tail, cc, vma, task);
 299
 300         return cc->nr;
 301 }
 302
 303 #ifdef CONFIG_ARM64
 304 static u32 __user *
 305 user_backtrace_compat(struct pt_regs *regs,
 306                       u32 __user *tail,
 307                       struct quadd_callchain *cc,
 308                       struct vm_area_struct *stack_vma,
 309                       struct task_struct *task)
 310 {
 311         int nr_added;
 312         u32 value, value_lr = 0, value_fp = 0;
 313         u32 __user *fp_prev = NULL;
 314
 315         if (!is_vma_addr((unsigned long)tail, stack_vma, sizeof(*tail)))
 316                 return NULL;
 317
 318         if (__copy_from_user_inatomic(&value, tail, sizeof(value))) {
 319                 cc->unw_rc = QUADD_URC_EACCESS;
 320                 return NULL;
 321         }
 322
 323         if (is_vma_addr(value, stack_vma, sizeof(value))) {
 324                 /* gcc thumb/clang frame */
 325                 value_fp = value;
 326
 327                 if (!is_vma_addr((unsigned long)(tail + 1), stack_vma,
 328                     sizeof(*tail)))
 329                         return NULL;
 330
 331                 if (__copy_from_user_inatomic(&value_lr, tail + 1,
 332                                               sizeof(value_lr))) {
 333                         cc->unw_rc = QUADD_URC_EACCESS;
 334                         return NULL;
 335                 }
 336
 337                 cc->curr_fp = value_fp;
 338                 cc->curr_sp = (unsigned long)tail + sizeof(value_fp) * 2;
 339                 cc->curr_pc = value_lr;
 340         } else {
 341                 /* gcc arm frame */
 342                 if (__copy_from_user_inatomic(&value_fp, tail - 1,
 343                                               sizeof(value_fp))) {
 344                         cc->unw_rc = QUADD_URC_EACCESS;
 345                         return NULL;
 346                 }
 347
 348                 if (!is_vma_addr(value_fp, stack_vma, sizeof(value_fp)))
 349                         return NULL;
 350
 351                 cc->curr_fp = value_fp;
 352                 cc->curr_sp = (unsigned long)tail + sizeof(value_fp);
 353                 cc->curr_pc = value_lr = value;
 354         }
 355
 356         fp_prev = (u32 __user *)(unsigned long)value_fp;
 357         if (fp_prev <= tail)
 358                 return NULL;
 359
 360         nr_added = quadd_callchain_store(cc, value_lr, QUADD_UNW_TYPE_FP);
 361         if (nr_added == 0)
 362                 return NULL;
 363
 364         if (cc->unw_method == QUADD_UNW_METHOD_MIXED &&
 365             is_ex_entry_exist(regs, value_lr, task))
 366                 return NULL;
 367
 368         return fp_prev;
 369 }
 370
 371 static unsigned int
 372 get_user_callchain_fp_compat(struct pt_regs *regs,
 373                              struct quadd_callchain *cc,
 374                              struct task_struct *task)
 375 {
 376         u32 fp, sp, pc, reg;
 377         struct vm_area_struct *vma, *vma_pc = NULL;
 378         u32 __user *tail = NULL;
 379         struct mm_struct *mm = task->mm;
 380
 381         cc->nr = 0;
 382         cc->unw_rc = QUADD_URC_FP_INCORRECT;
 383
 384         if (!regs || !mm) {
 385                 cc->unw_rc = QUADD_URC_FAILURE;
 386                 return 0;
 387         }
 388
 389         sp = quadd_user_stack_pointer(regs);
 390         pc = instruction_pointer(regs);
 391         fp = quadd_get_user_frame_pointer(regs);
 392
 393         if (fp == 0 || fp < sp || fp & 0x3)
 394                 return 0;
 395
 396         vma = find_vma(mm, sp);
 397         if (!vma) {
 398                 cc->unw_rc = QUADD_URC_SP_INCORRECT;
 399                 return 0;
 400         }
 401
 402         if (!is_vma_addr(fp, vma, sizeof(fp)))
 403                 return 0;
 404
 405         if (probe_kernel_address((unsigned long)fp, reg)) {
 406                 pr_warn_once("%s: failed for address: %#x\n", __func__, fp);
 407                 cc->unw_rc = QUADD_URC_EACCESS;
 408                 return 0;
 409         }
 410
 411         pr_debug("sp/fp: %#x/%#x, pc/lr: %#x/%#x, *fp: %#x, stack: %#lx-%#lx\n",
 412                  sp, fp, pc, (u32)quadd_user_link_register(regs), reg,
 413                  vma->vm_start, vma->vm_end);
 414
 415         if (is_thumb_mode(regs)) {
 416                 if (reg <= fp || !is_vma_addr(reg, vma, sizeof(reg)))
 417                         return 0;
 418         } else if (reg > fp && is_vma_addr(reg, vma, sizeof(reg))) {
 419                 /* fp --> fp prev */
 420                 u32 value;
 421                 int read_lr = 0;
 422
 423                 if (is_vma_addr(fp + sizeof(u32), vma, sizeof(fp))) {
 424                         if (__copy_from_user_inatomic(
 425                                         &value,
 426                                         (u32 __user *)(fp + sizeof(u32)),
 427                                         sizeof(value))) {
 428                                 cc->unw_rc = QUADD_URC_EACCESS;
 429                                 return 0;
 430                         }
 431
 432                         vma_pc = find_vma(mm, pc);
 433                         read_lr = 1;
 434                 }
 435
 436                 if (!read_lr || !is_vma_addr(value, vma_pc, sizeof(value))) {
 437                         /* gcc: fp --> short frame tail (fp) */
 438                         int nr_added;
 439                         u32 lr = quadd_user_link_register(regs);
 440
 441                         nr_added = quadd_callchain_store(cc, lr,
 442                                                          QUADD_UNW_TYPE_FP);
 443                         if (nr_added == 0)
 444                                 return cc->nr;
 445
 446                         tail = (u32 __user *)(unsigned long)reg;
 447                 }
 448         }
 449
 450         if (!tail)
 451                 tail = (u32 __user *)(unsigned long)fp;
 452
 453         while (tail && !((unsigned long)tail & 0x3))
 454                 tail = user_backtrace_compat(regs, tail, cc, vma, task);
 455
 456         return cc->nr;
 457 }
 458
 459 static unsigned int
 460 __user_backtrace_compat(struct pt_regs *regs,
 461                         struct quadd_callchain *cc,
 462                         struct task_struct *task)
 463 {
 464         struct mm_struct *mm = task->mm;
 465         struct vm_area_struct *vma;
 466         u32 __user *tail;
 467
 468         cc->unw_rc = QUADD_URC_FP_INCORRECT;
 469
 470         if (!mm) {
 471                 cc->unw_rc = QUADD_URC_FAILURE;
 472                 return cc->nr;
 473         }
 474
 475         vma = find_vma(mm, cc->curr_sp);
 476         if (!vma) {
 477                 cc->unw_rc = QUADD_URC_SP_INCORRECT;
 478                 return cc->nr;
 479         }
 480
 481         tail = (u32 __user *)cc->curr_fp;
 482
 483         while (tail && !((unsigned long)tail & 0x3))
 484                 tail = user_backtrace_compat(regs, tail, cc, vma, task);
 485
 486         return cc->nr;
 487 }
 488
 489 #endif  /* CONFIG_ARM64 */
 490
 491 static unsigned int
 492 __get_user_callchain_fp(struct pt_regs *regs,
 493                       struct quadd_callchain *cc,
 494                       struct task_struct *task)
 495 {
 496         if (cc->nr > 0) {
 497                 if (cc->unw_rc == QUADD_URC_LEVEL_TOO_DEEP)
 498                         return cc->nr;
 499
 500 #ifdef CONFIG_ARM64
 501                 if (compat_user_mode(regs))
 502                         __user_backtrace_compat(regs, cc, task);
 503                 else
 504                         __user_backtrace(regs, cc, task);
 505 #else
 506                 __user_backtrace(regs, cc, task);
 507 #endif
 508
 509                 return cc->nr;
 510         }
 511
 512 #ifdef CONFIG_ARM64
 513         if (compat_user_mode(regs))
 514                 return get_user_callchain_fp_compat(regs, cc, task);
 515 #endif
 516
 517         return get_user_callchain_fp(regs, cc, task);
 518 }
 519
 520 static unsigned int
 521 get_user_callchain_ut(struct pt_regs *regs,
 522                       struct quadd_callchain *cc,
 523                       struct task_struct *task)
 524 {
 525         int nr_prev;
 526         unsigned long prev_sp;
 527
 528         do {
 529                 nr_prev = cc->nr;
 530                 prev_sp = cc->curr_sp;
 531
 532                 quadd_get_user_cc_dwarf(regs, cc, task);
 533                 if (nr_prev > 0 && cc->nr == nr_prev)
 534                         break;
 535
 536                 nr_prev = cc->nr;
 537
 538                 quadd_get_user_cc_arm32_ehabi(regs, cc, task);
 539         } while (nr_prev != cc->nr &&
 540                  (cc->nr <= 1 || cc->curr_sp > prev_sp));
 541
 542         return cc->nr;
 543 }
 544
 545 static unsigned int
 546 get_user_callchain_mixed(struct pt_regs *regs,
 547                       struct quadd_callchain *cc,
 548                       struct task_struct *task)
 549 {
 550         int nr_prev;
 551         unsigned long prev_sp;
 552
 553         do {
 554                 nr_prev = cc->nr;
 555                 prev_sp = cc->curr_sp;
 556
 557                 quadd_get_user_cc_dwarf(regs, cc, task);
 558                 quadd_get_user_cc_arm32_ehabi(regs, cc, task);
 559
 560                 if (nr_prev != cc->nr) {
 561                         if (cc->nr > 1 &&
 562                             cc->curr_sp <= prev_sp)
 563                                 break;
 564
 565                         continue;
 566                 }
 567
 568                 __get_user_callchain_fp(regs, cc, task);
 569         } while (nr_prev != cc->nr &&
 570                  (cc->nr <= 1 || cc->curr_sp > prev_sp));
 571
 572         return cc->nr;
 573 }
 574
 575 unsigned int
 576 quadd_get_user_callchain(struct pt_regs *regs,
 577                          struct quadd_callchain *cc,
 578                          struct quadd_ctx *ctx,
 579                          struct task_struct *task)
 580 {
 581         unsigned int method = cc->unw_method;
 582
 583         cc->nr = 0;
 584
 585         if (!regs) {
 586                 cc->unw_rc = QUADD_URC_FAILURE;
 587                 return 0;
 588         }
 589
 590         cc->curr_sp = 0;
 591         cc->curr_fp = 0;
 592         cc->curr_fp_thumb = 0;
 593         cc->curr_pc = 0;
 594
 595 #ifdef CONFIG_ARM64
 596         cc->cs_64 = compat_user_mode(regs) ? 0 : 1;
 597 #else
 598         cc->cs_64 = 0;
 599 #endif
 600
 601         cc->unw_rc = 0;
 602
 603         switch (method) {
 604         case QUADD_UNW_METHOD_FP:
 605                 __get_user_callchain_fp(regs, cc, task);
 606                 break;
 607
 608         case QUADD_UNW_METHOD_EHT:
 609                 get_user_callchain_ut(regs, cc, task);
 610                 break;
 611
 612         case QUADD_UNW_METHOD_MIXED:
 613                 get_user_callchain_mixed(regs, cc, task);
 614                 break;
 615
 616         case QUADD_UNW_METHOD_NONE:
 617         default:
 618                 break;
 619         }
 620
 621         return cc->nr;
 622 }