drivers/misc/tegra-profiler/backtrace.c

   1 /*
   2  * drivers/misc/tegra-profiler/backtrace.c
   3  *
   4  * Copyright (c) 2015, NVIDIA CORPORATION.  All rights reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  */
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/uaccess.h>
  20 #include <linux/sched.h>
  21 #include <linux/mm.h>
  22
  23 #include <linux/tegra_profiler.h>
  24
  25 #include "quadd.h"
  26 #include "backtrace.h"
  27 #include "eh_unwind.h"
  28 #include "dwarf_unwind.h"
  29 #include "hrt.h"
  30 #include "tegra.h"
  31
  32 static inline int
  33 is_table_unwinding(struct quadd_callchain *cc)
  34 {
  35         return cc->um.ut || cc->um.dwarf;
  36 }
  37
  38 unsigned long
  39 quadd_user_stack_pointer(struct pt_regs *regs)
  40 {
  41 #ifdef CONFIG_ARM64
  42         if (compat_user_mode(regs))
  43                 return regs->compat_sp;
  44 #endif
  45
  46         return user_stack_pointer(regs);
  47 }
  48
  49 unsigned long
  50 quadd_get_user_frame_pointer(struct pt_regs *regs)
  51 {
  52         unsigned long fp;
  53
  54 #ifdef CONFIG_ARM64
  55         if (compat_user_mode(regs))
  56                 fp = is_thumb_mode(regs) ?
  57                         regs->compat_usr(7) : regs->compat_usr(11);
  58         else
  59                 fp = regs->regs[29];
  60 #else
  61         fp = is_thumb_mode(regs) ? regs->ARM_r7 : regs->ARM_fp;
  62 #endif
  63         return fp;
  64 }
  65
  66 unsigned long
  67 quadd_user_link_register(struct pt_regs *regs)
  68 {
  69 #ifdef CONFIG_ARM64
  70         return compat_user_mode(regs) ?
  71                 regs->compat_lr : regs->regs[30];
  72 #else
  73         return regs->ARM_lr;
  74 #endif
  75 }
  76
  77 static inline void
  78 put_unw_type(u32 *p, int bt_idx, unsigned int type)
  79 {
  80         int word_idx, shift;
  81
  82         word_idx = bt_idx / 8;
  83         shift = (bt_idx % 8) * 4;
  84
  85         *(p + word_idx) &= ~(0x0f << shift);
  86         *(p + word_idx) |= (type & 0x0f) << shift;
  87 }
  88
  89 int
  90 quadd_callchain_store(struct quadd_callchain *cc,
  91                       unsigned long ip, unsigned int type)
  92 {
  93         unsigned long low_addr = cc->hrt->low_addr;
  94
  95         if (ip < low_addr || !validate_pc_addr(ip, sizeof(unsigned long))) {
  96                 cc->urc_fp = QUADD_URC_PC_INCORRECT;
  97                 return 0;
  98         }
  99
 100         if (cc->nr >= QUADD_MAX_STACK_DEPTH) {
 101                 cc->urc_fp = QUADD_URC_LEVEL_TOO_DEEP;
 102                 return 0;
 103         }
 104
 105         put_unw_type(cc->types, cc->nr, type);
 106
 107         if (cc->cs_64)
 108                 cc->ip_64[cc->nr++] = ip;
 109         else
 110                 cc->ip_32[cc->nr++] = ip;
 111
 112         return 1;
 113 }
 114
 115 static int
 116 is_ex_entry_exist(struct pt_regs *regs,
 117                   unsigned long addr,
 118                   struct task_struct *task)
 119 {
 120         return quadd_is_ex_entry_exist_dwarf(regs, addr, task) ||
 121                quadd_is_ex_entry_exist_arm32_ehabi(regs, addr, task);
 122 }
 123
 124 static unsigned long __user *
 125 user_backtrace(struct pt_regs *regs,
 126                unsigned long __user *tail,
 127                struct quadd_callchain *cc,
 128                struct vm_area_struct *stack_vma,
 129                struct task_struct *task)
 130 {
 131         int nr_added;
 132         unsigned long value, value_lr = 0, value_fp = 0;
 133         unsigned long __user *fp_prev = NULL;
 134
 135         if (!is_vma_addr((unsigned long)tail, stack_vma, sizeof(*tail)))
 136                 return NULL;
 137
 138         if (__copy_from_user_inatomic(&value, tail, sizeof(unsigned long))) {
 139                 cc->urc_fp = QUADD_URC_EACCESS;
 140                 return NULL;
 141         }
 142
 143         if (is_vma_addr(value, stack_vma, sizeof(value))) {
 144                 /* gcc thumb/clang frame */
 145                 value_fp = value;
 146
 147                 if (!is_vma_addr((unsigned long)(tail + 1), stack_vma,
 148                     sizeof(*tail)))
 149                         return NULL;
 150
 151                 if (__copy_from_user_inatomic(&value_lr, tail + 1,
 152                                               sizeof(value_lr))) {
 153                         cc->urc_fp = QUADD_URC_EACCESS;
 154                         return NULL;
 155                 }
 156
 157                 cc->curr_fp = value_fp;
 158                 cc->curr_sp = (unsigned long)tail + sizeof(value_fp) * 2;
 159                 cc->curr_pc = value_lr;
 160         } else {
 161                 /* gcc arm frame */
 162                 if (__copy_from_user_inatomic(&value_fp, tail - 1,
 163                                               sizeof(value_fp))) {
 164                         cc->urc_fp = QUADD_URC_EACCESS;
 165                         return NULL;
 166                 }
 167
 168                 if (!is_vma_addr(value_fp, stack_vma, sizeof(value_fp)))
 169                         return NULL;
 170
 171                 cc->curr_fp = value_fp;
 172                 cc->curr_sp = (unsigned long)tail + sizeof(value_fp);
 173                 cc->curr_pc = value_lr = value;
 174         }
 175
 176         fp_prev = (unsigned long __user *)value_fp;
 177         if (fp_prev <= tail)
 178                 return NULL;
 179
 180         nr_added = quadd_callchain_store(cc, value_lr, QUADD_UNW_TYPE_FP);
 181         if (nr_added == 0)
 182                 return NULL;
 183
 184         if (is_table_unwinding(cc) &&
 185             is_ex_entry_exist(regs, value_lr, task))
 186                 return NULL;
 187
 188         return fp_prev;
 189 }
 190
 191 static unsigned int
 192 get_user_callchain_fp(struct pt_regs *regs,
 193                       struct quadd_callchain *cc,
 194                       struct task_struct *task)
 195 {
 196         unsigned long fp, sp, pc, reg;
 197         struct vm_area_struct *vma, *vma_pc = NULL;
 198         unsigned long __user *tail = NULL;
 199         struct mm_struct *mm = task->mm;
 200
 201         cc->nr = 0;
 202         cc->urc_fp = QUADD_URC_FP_INCORRECT;
 203
 204         if (!regs || !mm) {
 205                 cc->urc_fp = QUADD_URC_FAILURE;
 206                 return 0;
 207         }
 208
 209         sp = quadd_user_stack_pointer(regs);
 210         pc = instruction_pointer(regs);
 211         fp = quadd_get_user_frame_pointer(regs);
 212
 213         if (fp == 0 || fp < sp || fp & 0x3)
 214                 return 0;
 215
 216         vma = find_vma(mm, sp);
 217         if (!vma) {
 218                 cc->urc_fp = QUADD_URC_SP_INCORRECT;
 219                 return 0;
 220         }
 221
 222         if (!is_vma_addr(fp, vma, sizeof(fp)))
 223                 return 0;
 224
 225         if (probe_kernel_address(fp, reg)) {
 226                 pr_warn_once("%s: failed for address: %#lx\n", __func__, fp);
 227                 cc->urc_fp = QUADD_URC_EACCESS;
 228                 return 0;
 229         }
 230
 231         pr_debug("sp/fp: %#lx/%#lx, pc/lr: %#lx/%#lx, *fp: %#lx, stack: %#lx-%#lx\n",
 232                  sp, fp, pc, quadd_user_link_register(regs), reg,
 233                  vma->vm_start, vma->vm_end);
 234
 235         if (is_thumb_mode(regs)) {
 236                 if (reg <= fp || !is_vma_addr(reg, vma, sizeof(reg)))
 237                         return 0;
 238         } else if (reg > fp && is_vma_addr(reg, vma, sizeof(reg))) {
 239                 /* fp --> fp prev */
 240                 unsigned long value;
 241                 int read_lr = 0;
 242
 243                 if (is_vma_addr(fp + sizeof(unsigned long), vma, sizeof(fp))) {
 244                         if (__copy_from_user_inatomic(
 245                                         &value,
 246                                         (unsigned long __user *)fp + 1,
 247                                         sizeof(unsigned long))) {
 248                                 cc->urc_fp = QUADD_URC_EACCESS;
 249                                 return 0;
 250                         }
 251
 252                         vma_pc = find_vma(mm, pc);
 253                         read_lr = 1;
 254                 }
 255
 256                 if (!read_lr || !is_vma_addr(value, vma_pc, sizeof(value))) {
 257                         /* gcc: fp --> short frame tail (fp) */
 258                         int nr_added;
 259                         unsigned long lr = quadd_user_link_register(regs);
 260
 261                         nr_added = quadd_callchain_store(cc, lr,
 262                                                          QUADD_UNW_TYPE_FP);
 263                         if (nr_added == 0)
 264                                 return cc->nr;
 265
 266                         tail = (unsigned long __user *)reg;
 267                 }
 268         }
 269
 270         if (!tail)
 271                 tail = (unsigned long __user *)fp;
 272
 273         while (tail && !((unsigned long)tail & 0x3))
 274                 tail = user_backtrace(regs, tail, cc, vma, task);
 275
 276         return cc->nr;
 277 }
 278
 279 static unsigned int
 280 __user_backtrace(struct pt_regs *regs,
 281                  struct quadd_callchain *cc,
 282                  struct task_struct *task)
 283 {
 284         struct mm_struct *mm = task->mm;
 285         struct vm_area_struct *vma;
 286         unsigned long __user *tail;
 287
 288         cc->urc_fp = QUADD_URC_FP_INCORRECT;
 289
 290         if (!mm) {
 291                 cc->urc_fp = QUADD_URC_FAILURE;
 292                 return cc->nr;
 293         }
 294
 295         vma = find_vma(mm, cc->curr_sp);
 296         if (!vma) {
 297                 cc->urc_fp = QUADD_URC_SP_INCORRECT;
 298                 return cc->nr;
 299         }
 300
 301         tail = (unsigned long __user *)cc->curr_fp;
 302
 303         while (tail && !((unsigned long)tail & 0x3))
 304                 tail = user_backtrace(regs, tail, cc, vma, task);
 305
 306         return cc->nr;
 307 }
 308
 309 #ifdef CONFIG_ARM64
 310 static u32 __user *
 311 user_backtrace_compat(struct pt_regs *regs,
 312                       u32 __user *tail,
 313                       struct quadd_callchain *cc,
 314                       struct vm_area_struct *stack_vma,
 315                       struct task_struct *task)
 316 {
 317         int nr_added;
 318         u32 value, value_lr = 0, value_fp = 0;
 319         u32 __user *fp_prev = NULL;
 320
 321         if (!is_vma_addr((unsigned long)tail, stack_vma, sizeof(*tail)))
 322                 return NULL;
 323
 324         if (__copy_from_user_inatomic(&value, tail, sizeof(value))) {
 325                 cc->urc_fp = QUADD_URC_EACCESS;
 326                 return NULL;
 327         }
 328
 329         if (is_vma_addr(value, stack_vma, sizeof(value))) {
 330                 /* gcc thumb/clang frame */
 331                 value_fp = value;
 332
 333                 if (!is_vma_addr((unsigned long)(tail + 1), stack_vma,
 334                     sizeof(*tail)))
 335                         return NULL;
 336
 337                 if (__copy_from_user_inatomic(&value_lr, tail + 1,
 338                                               sizeof(value_lr))) {
 339                         cc->urc_fp = QUADD_URC_EACCESS;
 340                         return NULL;
 341                 }
 342
 343                 cc->curr_fp = value_fp;
 344                 cc->curr_sp = (unsigned long)tail + sizeof(value_fp) * 2;
 345                 cc->curr_pc = value_lr;
 346         } else {
 347                 /* gcc arm frame */
 348                 if (__copy_from_user_inatomic(&value_fp, tail - 1,
 349                                               sizeof(value_fp))) {
 350                         cc->urc_fp = QUADD_URC_EACCESS;
 351                         return NULL;
 352                 }
 353
 354                 if (!is_vma_addr(value_fp, stack_vma, sizeof(value_fp)))
 355                         return NULL;
 356
 357                 cc->curr_fp = value_fp;
 358                 cc->curr_sp = (unsigned long)tail + sizeof(value_fp);
 359                 cc->curr_pc = value_lr = value;
 360         }
 361
 362         fp_prev = (u32 __user *)(unsigned long)value_fp;
 363         if (fp_prev <= tail)
 364                 return NULL;
 365
 366         nr_added = quadd_callchain_store(cc, value_lr, QUADD_UNW_TYPE_FP);
 367         if (nr_added == 0)
 368                 return NULL;
 369
 370         if (is_table_unwinding(cc) &&
 371             is_ex_entry_exist(regs, value_lr, task))
 372                 return NULL;
 373
 374         return fp_prev;
 375 }
 376
 377 static unsigned int
 378 get_user_callchain_fp_compat(struct pt_regs *regs,
 379                              struct quadd_callchain *cc,
 380                              struct task_struct *task)
 381 {
 382         u32 fp, sp, pc, reg;
 383         struct vm_area_struct *vma, *vma_pc = NULL;
 384         u32 __user *tail = NULL;
 385         struct mm_struct *mm = task->mm;
 386
 387         cc->nr = 0;
 388         cc->urc_fp = QUADD_URC_FP_INCORRECT;
 389
 390         if (!regs || !mm) {
 391                 cc->urc_fp = QUADD_URC_FAILURE;
 392                 return 0;
 393         }
 394
 395         sp = quadd_user_stack_pointer(regs);
 396         pc = instruction_pointer(regs);
 397         fp = quadd_get_user_frame_pointer(regs);
 398
 399         if (fp == 0 || fp < sp || fp & 0x3)
 400                 return 0;
 401
 402         vma = find_vma(mm, sp);
 403         if (!vma) {
 404                 cc->urc_fp = QUADD_URC_SP_INCORRECT;
 405                 return 0;
 406         }
 407
 408         if (!is_vma_addr(fp, vma, sizeof(fp)))
 409                 return 0;
 410
 411         if (probe_kernel_address((unsigned long)fp, reg)) {
 412                 pr_warn_once("%s: failed for address: %#x\n", __func__, fp);
 413                 cc->urc_fp = QUADD_URC_EACCESS;
 414                 return 0;
 415         }
 416
 417         pr_debug("sp/fp: %#x/%#x, pc/lr: %#x/%#x, *fp: %#x, stack: %#lx-%#lx\n",
 418                  sp, fp, pc, (u32)quadd_user_link_register(regs), reg,
 419                  vma->vm_start, vma->vm_end);
 420
 421         if (is_thumb_mode(regs)) {
 422                 if (reg <= fp || !is_vma_addr(reg, vma, sizeof(reg)))
 423                         return 0;
 424         } else if (reg > fp && is_vma_addr(reg, vma, sizeof(reg))) {
 425                 /* fp --> fp prev */
 426                 u32 value;
 427                 int read_lr = 0;
 428
 429                 if (is_vma_addr(fp + sizeof(u32), vma, sizeof(fp))) {
 430                         if (__copy_from_user_inatomic(
 431                                         &value,
 432                                         (u32 __user *)(fp + sizeof(u32)),
 433                                         sizeof(value))) {
 434                                 cc->urc_fp = QUADD_URC_EACCESS;
 435                                 return 0;
 436                         }
 437
 438                         vma_pc = find_vma(mm, pc);
 439                         read_lr = 1;
 440                 }
 441
 442                 if (!read_lr || !is_vma_addr(value, vma_pc, sizeof(value))) {
 443                         /* gcc: fp --> short frame tail (fp) */
 444                         int nr_added;
 445                         u32 lr = quadd_user_link_register(regs);
 446
 447                         nr_added = quadd_callchain_store(cc, lr,
 448                                                          QUADD_UNW_TYPE_FP);
 449                         if (nr_added == 0)
 450                                 return cc->nr;
 451
 452                         tail = (u32 __user *)(unsigned long)reg;
 453                 }
 454         }
 455
 456         if (!tail)
 457                 tail = (u32 __user *)(unsigned long)fp;
 458
 459         while (tail && !((unsigned long)tail & 0x3))
 460                 tail = user_backtrace_compat(regs, tail, cc, vma, task);
 461
 462         return cc->nr;
 463 }
 464
 465 static unsigned int
 466 __user_backtrace_compat(struct pt_regs *regs,
 467                         struct quadd_callchain *cc,
 468                         struct task_struct *task)
 469 {
 470         struct mm_struct *mm = task->mm;
 471         struct vm_area_struct *vma;
 472         u32 __user *tail;
 473
 474         cc->urc_fp = QUADD_URC_FP_INCORRECT;
 475
 476         if (!mm) {
 477                 cc->urc_fp = QUADD_URC_FAILURE;
 478                 return cc->nr;
 479         }
 480
 481         vma = find_vma(mm, cc->curr_sp);
 482         if (!vma) {
 483                 cc->urc_fp = QUADD_URC_SP_INCORRECT;
 484                 return cc->nr;
 485         }
 486
 487         tail = (u32 __user *)cc->curr_fp;
 488
 489         while (tail && !((unsigned long)tail & 0x3))
 490                 tail = user_backtrace_compat(regs, tail, cc, vma, task);
 491
 492         return cc->nr;
 493 }
 494
 495 #endif  /* CONFIG_ARM64 */
 496
 497 static unsigned int
 498 __get_user_callchain_fp(struct pt_regs *regs,
 499                       struct quadd_callchain *cc,
 500                       struct task_struct *task)
 501 {
 502         if (cc->nr > 0) {
 503                 if (cc->urc_fp == QUADD_URC_LEVEL_TOO_DEEP)
 504                         return cc->nr;
 505
 506 #ifdef CONFIG_ARM64
 507                 if (compat_user_mode(regs))
 508                         __user_backtrace_compat(regs, cc, task);
 509                 else
 510                         __user_backtrace(regs, cc, task);
 511 #else
 512                 __user_backtrace(regs, cc, task);
 513 #endif
 514
 515                 return cc->nr;
 516         }
 517
 518 #ifdef CONFIG_ARM64
 519         if (compat_user_mode(regs))
 520                 return get_user_callchain_fp_compat(regs, cc, task);
 521 #endif
 522
 523         return get_user_callchain_fp(regs, cc, task);
 524 }
 525
 526 static unsigned int
 527 get_user_callchain_mixed(struct pt_regs *regs,
 528                       struct quadd_callchain *cc,
 529                       struct task_struct *task)
 530 {
 531         int nr_prev;
 532         unsigned long prev_sp;
 533         struct quadd_unw_methods *um = &cc->um;
 534
 535         do {
 536                 nr_prev = cc->nr;
 537                 prev_sp = cc->curr_sp;
 538
 539                 if (um->dwarf)
 540                         quadd_get_user_cc_dwarf(regs, cc, task);
 541                 if (um->ut)
 542                         quadd_get_user_cc_arm32_ehabi(regs, cc, task);
 543
 544                 if (nr_prev != cc->nr) {
 545                         if (cc->nr > 1 &&
 546                             cc->curr_sp <= prev_sp)
 547                                 break;
 548
 549                         continue;
 550                 }
 551
 552                 if (um->fp)
 553                         __get_user_callchain_fp(regs, cc, task);
 554         } while (nr_prev != cc->nr &&
 555                  (cc->nr <= 1 || cc->curr_sp > prev_sp));
 556
 557         return cc->nr;
 558 }
 559
 560 unsigned int
 561 quadd_get_user_callchain(struct pt_regs *regs,
 562                          struct quadd_callchain *cc,
 563                          struct quadd_ctx *ctx,
 564                          struct task_struct *task)
 565 {
 566         cc->nr = 0;
 567
 568         if (!regs) {
 569                 cc->urc_fp = QUADD_URC_FAILURE;
 570                 cc->urc_ut = QUADD_URC_FAILURE;
 571                 cc->urc_dwarf = QUADD_URC_FAILURE;
 572                 return 0;
 573         }
 574
 575         cc->curr_sp = 0;
 576         cc->curr_fp = 0;
 577         cc->curr_fp_thumb = 0;
 578         cc->curr_pc = 0;
 579
 580 #ifdef CONFIG_ARM64
 581         cc->cs_64 = compat_user_mode(regs) ? 0 : 1;
 582 #else
 583         cc->cs_64 = 0;
 584 #endif
 585
 586         cc->urc_fp = QUADD_URC_NONE;
 587         cc->urc_ut = QUADD_URC_NONE;
 588         cc->urc_dwarf = QUADD_URC_NONE;
 589
 590         get_user_callchain_mixed(regs, cc, task);
 591
 592         return cc->nr;
 593 }