drivers/misc/tegra-profiler/dwarf_unwind.c

   1 /*
   2  * drivers/misc/tegra-profiler/dwarf_unwind.c
   3  *
   4  * Copyright (c) 2015, NVIDIA CORPORATION.  All rights reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  */
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/mm.h>
  20 #include <linux/sched.h>
  21 #include <linux/slab.h>
  22 #include <linux/uaccess.h>
  23 #include <linux/err.h>
  24
  25 #include <asm/unaligned.h>
  26
  27 #include <linux/tegra_profiler.h>
  28
  29 #include "comm.h"
  30 #include "backtrace.h"
  31 #include "eh_unwind.h"
  32 #include "dwarf.h"
  33 #include "dwarf_unwind.h"
  34
  35 enum {
  36         DW_WHERE_UNDEF,         /* register isn't saved at all */
  37         DW_WHERE_SAME,          /* register has same value as in prev. frame */
  38         DW_WHERE_CFAREL,        /* register saved at CFA-relative address */
  39         DW_WHERE_REG,           /* register saved in another register */
  40         DW_WHERE_EXPR,          /* register saved */
  41         DW_WHERE_VAL_OFFSET,    /* value offset */
  42         DW_WHERE_VAL_EXPR,      /* register has computed value */
  43 };
  44
  45 #define QUADD_AARCH64_REGISTERS 32
  46 #define QUADD_AARCH32_REGISTERS 16
  47
  48 #define QUADD_NUM_REGS QUADD_AARCH64_REGISTERS
  49
  50 enum regs32 {
  51         ARM32_FP_THUMB = 7,
  52         ARM32_FP = 11,
  53
  54         ARM32_SP = 13,
  55         ARM32_LR = 14,
  56         ARM32_PC = 15
  57 };
  58
  59 enum regs64 {
  60         ARM64_FP = 29,
  61         ARM64_LR = 30,
  62         ARM64_SP = 31,
  63 };
  64
  65 enum {
  66         DW_MODE_ARM32,
  67         DW_MODE_ARM64,
  68 };
  69
  70 union dw_loc {
  71         unsigned long reg;
  72         long offset;
  73         const unsigned char *exp;
  74 };
  75
  76 struct reg_info {
  77         int where;
  78         union dw_loc loc;
  79 };
  80
  81 enum {
  82         DW_CFA_UNSET,
  83         DW_CFA_REG_OFFSET,
  84         DW_CFA_EXP,
  85 };
  86
  87 struct dw_eh_frame_hdr {
  88         unsigned char version;
  89         unsigned char eh_frame_ptr_enc;
  90         unsigned char fde_count_enc;
  91         unsigned char table_enc;
  92 };
  93
  94 struct dw_fde_table {
  95         s32 initial_loc;
  96         s32 fde;
  97 };
  98
  99 struct regs_state {
 100         struct reg_info reg[QUADD_NUM_REGS];
 101
 102         long cfa_offset;
 103         int cfa_register;
 104
 105         unsigned char *cfa_expr;
 106         unsigned int cfa_expr_len;
 107
 108         int cfa_how;
 109 };
 110
 111 #define DW_MAX_RS_STACK_DEPTH   8
 112
 113 struct dwarf_cpu_context {
 114         struct regs_state rs_stack[DW_MAX_RS_STACK_DEPTH];
 115         int depth;
 116
 117         int dw_ptr_size;
 118 };
 119
 120 struct quadd_dwarf_context {
 121         struct dwarf_cpu_context __percpu *cpu_ctx;
 122         atomic_t started;
 123 };
 124
 125 struct stackframe {
 126         unsigned long pc;
 127         unsigned long vregs[QUADD_NUM_REGS];
 128
 129         struct regs_state rs;
 130         struct regs_state rs_initial;
 131
 132         unsigned long cfa;
 133
 134         int mode;
 135 };
 136
 137 struct dw_cie {
 138         unsigned long offset;
 139         unsigned long length;
 140
 141         unsigned char *aug_string;
 142         unsigned long aug_size;
 143
 144         unsigned char fde_encoding;
 145         unsigned char lsda_encoding;
 146
 147         unsigned long code_align_factor;
 148         long data_align_factor;
 149
 150         unsigned int initial_insn_len;
 151         unsigned char *initial_insn;
 152
 153         int z_aug;
 154         unsigned int retaddr_reg;
 155         void *personality;
 156
 157         unsigned char *data;
 158 };
 159
 160 struct dw_fde {
 161         unsigned long offset;
 162         unsigned long length;
 163
 164         unsigned long cie_pointer;
 165         struct dw_cie *cie;
 166
 167         unsigned long initial_location;
 168         unsigned long address_range;
 169
 170         unsigned int insn_length;
 171         unsigned char *instructions;
 172
 173         unsigned char *data;
 174 };
 175
 176
 177 struct eh_sec_data {
 178         size_t length;
 179         unsigned char *data;
 180 };
 181
 182 #define read_user_data(addr, retval)                            \
 183 ({                                                              \
 184         long ret;                                               \
 185                                                                 \
 186         pagefault_disable();                                    \
 187         ret = __get_user(retval, addr);                         \
 188         pagefault_enable();                                     \
 189                                                                 \
 190         if (ret) {                                              \
 191                 pr_debug("%s: failed for address: %p\n",        \
 192                          __func__, addr);                       \
 193                 ret = -QUADD_URC_EACCESS;                       \
 194         }                                                       \
 195                                                                 \
 196         ret;                                                    \
 197 })
 198
 199 static struct quadd_dwarf_context ctx;
 200
 201 static inline int regnum_sp(int mode)
 202 {
 203         return (mode == DW_MODE_ARM32) ?
 204                 ARM32_SP : ARM64_SP;
 205 }
 206
 207 static inline int regnum_fp(int mode)
 208 {
 209         return (mode == DW_MODE_ARM32) ?
 210                 ARM32_FP : ARM64_FP;
 211 }
 212
 213 static inline int regnum_lr(int mode)
 214 {
 215         return (mode == DW_MODE_ARM32) ?
 216                 ARM32_LR : ARM64_LR;
 217 }
 218
 219 static inline unsigned long
 220 get_user_reg_size(int mode)
 221 {
 222         return (mode == DW_MODE_ARM32) ?
 223                 sizeof(u32) : sizeof(u64);
 224 }
 225
 226 static inline int
 227 get_secid_frame(int is_eh)
 228 {
 229         return is_eh ?
 230                 QUADD_SEC_TYPE_EH_FRAME :
 231                 QUADD_SEC_TYPE_DEBUG_FRAME;
 232 }
 233
 234 static inline int
 235 get_secid_frame_hdr(int is_eh)
 236 {
 237         return is_eh ?
 238                 QUADD_SEC_TYPE_EH_FRAME_HDR :
 239                 QUADD_SEC_TYPE_DEBUG_FRAME_HDR;
 240 }
 241
 242 static inline int
 243 is_frame_present(struct ex_region_info *ri, int is_eh)
 244 {
 245         struct extab_info *ti, *ti_hdr;
 246
 247         ti = &ri->ex_sec[get_secid_frame(is_eh)];
 248         ti_hdr = &ri->ex_sec[get_secid_frame_hdr(is_eh)];
 249
 250         return (ti->length && ti_hdr->length) ? 1 : 0;
 251 }
 252
 253 static inline int
 254 validate_addr(struct ex_region_info *ri,
 255               unsigned long addr,
 256               unsigned long nbytes,
 257               int st)
 258 {
 259         struct extab_info *ti;
 260         struct quadd_mmap_area *mmap;
 261         unsigned long start, end;
 262
 263         mmap = ri->mmap;
 264
 265         ti = &ri->ex_sec[st];
 266
 267         start = (unsigned long)mmap->data + ti->mmap_offset;
 268         end = start + ti->length;
 269
 270         if (unlikely(addr < start || addr > end - nbytes)) {
 271                 pr_err_once("%s: error: addr: %#lx, len: %ld, data: %#lx-%#lx\n",
 272                             __func__, addr, nbytes, start, end);
 273                 return 0;
 274         }
 275
 276         return 1;
 277 }
 278
 279 static inline u8
 280 read_mmap_data_u8(struct ex_region_info *ri,
 281                   const u8 *addr, int st, long *err)
 282 {
 283         unsigned long a = (unsigned long)addr;
 284
 285         if (unlikely(!validate_addr(ri, a, sizeof(*addr), st))) {
 286                 *err = -QUADD_URC_EACCESS;
 287                 return 0;
 288         }
 289
 290         *err = 0;
 291         return *addr;
 292 }
 293
 294 static inline u16
 295 read_mmap_data_u16(struct ex_region_info *ri,
 296                    const u16 *addr, int st, long *err)
 297 {
 298         unsigned long a = (unsigned long)addr;
 299
 300         if (unlikely(!validate_addr(ri, a, sizeof(*addr), st))) {
 301                 *err = -QUADD_URC_EACCESS;
 302                 return 0;
 303         }
 304
 305         *err = 0;
 306
 307         return get_unaligned(addr);
 308 }
 309
 310 static inline s16
 311 read_mmap_data_s16(struct ex_region_info *ri,
 312                    const s16 *addr, int st, long *err)
 313 {
 314         unsigned long a = (unsigned long)addr;
 315
 316         if (unlikely(!validate_addr(ri, a, sizeof(*addr), st))) {
 317                 *err = -QUADD_URC_EACCESS;
 318                 return 0;
 319         }
 320
 321         *err = 0;
 322
 323         return get_unaligned(addr);
 324 }
 325
 326 static inline u32
 327 read_mmap_data_u32(struct ex_region_info *ri,
 328                    const u32 *addr, int st, long *err)
 329 {
 330         unsigned long a = (unsigned long)addr;
 331
 332         if (unlikely(!validate_addr(ri, a, sizeof(*addr), st))) {
 333                 *err = -QUADD_URC_EACCESS;
 334                 return 0;
 335         }
 336
 337         *err = 0;
 338
 339         return get_unaligned(addr);
 340 }
 341
 342 static inline s32
 343 read_mmap_data_s32(struct ex_region_info *ri,
 344                    const s32 *addr, int st, long *err)
 345 {
 346         unsigned long a = (unsigned long)addr;
 347
 348         if (unlikely(!validate_addr(ri, a, sizeof(*addr), st))) {
 349                 *err = -QUADD_URC_EACCESS;
 350                 return 0;
 351         }
 352
 353         *err = 0;
 354
 355         return get_unaligned(addr);
 356 }
 357
 358 static inline s64
 359 read_mmap_data_s64(struct ex_region_info *ri,
 360                    const s64 *addr, int st, long *err)
 361 {
 362         unsigned long a = (unsigned long)addr;
 363
 364         if (unlikely(!validate_addr(ri, a, sizeof(*addr), st))) {
 365                 *err = -QUADD_URC_EACCESS;
 366                 return 0;
 367         }
 368
 369         *err = 0;
 370
 371         return get_unaligned(addr);
 372 }
 373
 374 static inline u64
 375 read_mmap_data_u64(struct ex_region_info *ri,
 376                    const u64 *addr, int st, long *err)
 377 {
 378         unsigned long a = (unsigned long)addr;
 379
 380         if (unlikely(!validate_addr(ri, a, sizeof(*addr), st))) {
 381                 *err = -QUADD_URC_EACCESS;
 382                 return 0;
 383         }
 384
 385         *err = 0;
 386
 387         return get_unaligned(addr);
 388 }
 389
 390 static inline unsigned long
 391 ex_addr_to_mmap_addr(unsigned long addr,
 392                      struct ex_region_info *ri, int st)
 393 {
 394         unsigned long offset;
 395         struct extab_info *ti;
 396
 397         ti = &ri->ex_sec[st];
 398         offset = addr - ti->addr;
 399
 400         return ti->mmap_offset + offset + (unsigned long)ri->mmap->data;
 401 }
 402
 403 static inline unsigned long
 404 mmap_addr_to_ex_addr(unsigned long addr,
 405                      struct ex_region_info *ri, int st)
 406 {
 407         unsigned long offset;
 408         struct extab_info *ti;
 409
 410         ti = &ri->ex_sec[st];
 411         offset = addr - ti->mmap_offset - (unsigned long)ri->mmap->data;
 412
 413         return ti->addr + offset;
 414 }
 415
 416 static inline int validate_regnum(struct regs_state *rs, int regnum)
 417 {
 418         if (unlikely(regnum >= ARRAY_SIZE(rs->reg))) {
 419                 pr_err_once("error: invalid reg: %d\n", regnum);
 420                 return 0;
 421         }
 422
 423         return 1;
 424 }
 425
 426 static inline void
 427 set_rule_offset(struct regs_state *rs, int regnum, int where, long offset)
 428 {
 429         struct reg_info *r;
 430
 431         if (!validate_regnum(rs, regnum))
 432                 return;
 433
 434         r = &rs->reg[regnum];
 435
 436         r->where = where;
 437         r->loc.offset = offset;
 438 }
 439
 440 static inline void
 441 set_rule_reg(struct regs_state *rs, int regnum, int where, unsigned long reg)
 442 {
 443         struct reg_info *r;
 444
 445         if (!validate_regnum(rs, regnum))
 446                 return;
 447
 448         r = &rs->reg[regnum];
 449
 450         r->where = where;
 451         r->loc.reg = reg;
 452 }
 453
 454 static inline void
 455 set_rule_exp(struct regs_state *rs, int regnum,
 456              int where, const unsigned char *exp)
 457 {
 458         struct reg_info *r;
 459
 460         if (!validate_regnum(rs, regnum))
 461                 return;
 462
 463         r = &rs->reg[regnum];
 464
 465         r->where = where;
 466         r->loc.exp = exp;
 467 }
 468
 469 static inline void
 470 set_rule(struct regs_state *rs, int regnum, int where, long value)
 471 {
 472         set_rule_offset(rs, regnum, where, value);
 473 }
 474
 475 static inline unsigned long
 476 dw_bst_get_initial_loc(const struct dw_fde_table *fi,
 477                        unsigned long data_base)
 478 {
 479         return data_base + fi->initial_loc;
 480 }
 481
 482 static inline unsigned long
 483 dw_bst_get_fde_addr(const struct dw_fde_table *fi,
 484                     unsigned long data_base)
 485 {
 486         return data_base + fi->fde;
 487 }
 488
 489 static inline unsigned long
 490 dwarf_read_uleb128(struct ex_region_info *ri,
 491                    unsigned char *addr,
 492                    unsigned long *ret,
 493                    int st,
 494                    long *err)
 495 {
 496         unsigned long result;
 497         unsigned char byte;
 498         int shift, count;
 499
 500         result = 0;
 501         shift = 0;
 502         count = 0;
 503
 504         while (1) {
 505                 byte = read_mmap_data_u8(ri, addr, st, err);
 506                 if (*err)
 507                         return 0;
 508
 509                 addr++;
 510                 count++;
 511
 512                 result |= (byte & 0x7f) << shift;
 513                 shift += 7;
 514
 515                 if (!(byte & 0x80))
 516                         break;
 517         }
 518
 519         *ret = result;
 520
 521         return count;
 522 }
 523
 524 static inline unsigned long
 525 dwarf_read_sleb128(struct ex_region_info *ri,
 526                    unsigned char *addr,
 527                    long *ret,
 528                    int st,
 529                    long *err)
 530 {
 531         unsigned char byte;
 532         long result, shift;
 533         int num_bits;
 534         int count;
 535
 536         result = 0;
 537         shift = 0;
 538         count = 0;
 539
 540         while (1) {
 541                 byte = read_mmap_data_u8(ri, addr, st, err);
 542                 if (*err)
 543                         return 0;
 544
 545                 addr++;
 546                 result |= (byte & 0x7f) << shift;
 547                 shift += 7;
 548                 count++;
 549
 550                 if (!(byte & 0x80))
 551                         break;
 552         }
 553
 554         num_bits = 8 * sizeof(result);
 555
 556         if ((shift < num_bits) && (byte & 0x40))
 557                 result |= (-1 << shift);
 558
 559         *ret = result;
 560
 561         return count;
 562 }
 563
 564 static inline unsigned int
 565 dw_cfa_opcode(unsigned int insn)
 566 {
 567         return insn & 0xc0;
 568 }
 569
 570 static inline unsigned int
 571 dw_cfa_operand(unsigned int insn)
 572 {
 573         return insn & 0x3f;
 574 }
 575
 576 static int
 577 dwarf_read_encoded_value(struct ex_region_info *ri,
 578                          void *addr,
 579                          void *pcrel_base,
 580                          unsigned long *val,
 581                          char encoding,
 582                          int st)
 583 {
 584         int dw_ptr_size, count = 0;
 585         long stmp = 0, err = 0;
 586         unsigned long utmp, res = 0;
 587         struct dwarf_cpu_context *cpu_ctx = this_cpu_ptr(ctx.cpu_ctx);
 588
 589         pr_debug("encoding: %#x\n", encoding);
 590
 591         dw_ptr_size = cpu_ctx->dw_ptr_size;
 592
 593         if (encoding == DW_EH_PE_omit) {
 594                 pr_debug("DW_EH_PE_omit\n");
 595
 596                 *val = 0;
 597                 return 0;
 598         } else if (encoding == DW_EH_PE_aligned) {
 599                 unsigned long aligned = ALIGN((unsigned long)addr,
 600                                               dw_ptr_size);
 601
 602                 pr_debug("DW_EH_PE_aligned\n");
 603
 604                 if (dw_ptr_size == 4) {
 605                         *val = read_mmap_data_u32(ri, (u32 *)aligned, st, &err);
 606                 } else if (dw_ptr_size == 8) {
 607                         *val = read_mmap_data_u64(ri, (u64 *)aligned, st, &err);
 608                 } else {
 609                         pr_err_once("%s: error: encoding\n", __func__);
 610                         return -QUADD_URC_TBL_IS_CORRUPT;
 611                 }
 612
 613                 if (err)
 614                         return err;
 615
 616                 return dw_ptr_size;
 617         }
 618
 619         switch (encoding & 0x0f) {
 620         case DW_EH_PE_absptr:
 621                 pr_debug("%s: absptr encoding\n", __func__);
 622
 623                 if (dw_ptr_size == 4) {
 624                         *val = read_mmap_data_u32(ri, (u32 *)addr, st, &err);
 625                 } else if (dw_ptr_size == 8) {
 626                         *val = read_mmap_data_u64(ri, (u64 *)addr, st, &err);
 627                 } else {
 628                         pr_err_once("error: wrong dwarf size\n");
 629                         return -QUADD_URC_UNHANDLED_INSTRUCTION;
 630                 }
 631
 632                 if (err)
 633                         return err;
 634
 635                 return dw_ptr_size;
 636
 637         case DW_EH_PE_sdata2:
 638         case DW_EH_PE_udata2:
 639                 pr_debug("encoding: DW_EH_PE_sdata2\n");
 640                 stmp = read_mmap_data_s16(ri, (s16 *)addr, st, &err);
 641                 if (err)
 642                         return err;
 643
 644                 count += sizeof(s16);
 645                 break;
 646
 647         case DW_EH_PE_sdata4:
 648         case DW_EH_PE_udata4:
 649                 pr_debug("encoding: DW_EH_PE_udata4/sdata4\n");
 650                 stmp = read_mmap_data_s32(ri, (s32 *)addr, st, &err);
 651                 if (err)
 652                         return err;
 653
 654                 count += sizeof(s32);
 655                 break;
 656
 657         case DW_EH_PE_sdata8:
 658         case DW_EH_PE_udata8:
 659                 pr_debug("encoding: DW_EH_PE_udata8\n");
 660                 stmp = read_mmap_data_s64(ri, (s64 *)addr, st, &err);
 661                 if (err)
 662                         return err;
 663
 664                 count += sizeof(s64);
 665                 break;
 666
 667         case DW_EH_PE_uleb128:
 668                 pr_debug("encoding: DW_EH_PE_uleb128\n");
 669                 count += dwarf_read_uleb128(ri, addr, &utmp, st, &err);
 670                 if (err)
 671                         return err;
 672
 673                 stmp = utmp;
 674                 break;
 675
 676         case DW_EH_PE_sleb128:
 677                 pr_debug("encoding: DW_EH_PE_sleb128\n");
 678                 count += dwarf_read_sleb128(ri, addr, &stmp, st, &err);
 679                 if (err)
 680                         return err;
 681
 682                 break;
 683
 684         default:
 685                 pr_warn_once("%s: warning: encoding: %#x\n",
 686                              __func__, encoding & 0x0f);
 687                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
 688         }
 689
 690         switch (encoding & 0x70) {
 691         case DW_EH_PE_absptr:
 692                 pr_debug("DW_EH_PE_absptr\n");
 693                 res = stmp;
 694                 break;
 695
 696         case DW_EH_PE_pcrel:
 697                 pr_debug("DW_EH_PE_pcrel, pcrel_base: %p, stmp: %ld\n",
 698                          pcrel_base, stmp);
 699                 res = (unsigned long)pcrel_base + stmp;
 700                 break;
 701
 702         case DW_EH_PE_textrel:
 703                 pr_warn_once("warning: DW_EH_PE_textrel\n");
 704                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
 705
 706         case DW_EH_PE_datarel:
 707                 pr_warn_once("warning: DW_EH_PE_datarel\n");
 708                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
 709
 710         case DW_EH_PE_funcrel:
 711                 pr_warn_once("warning: DW_EH_PE_funcrel\n");
 712                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
 713
 714         default:
 715                 pr_warn_once("%s: warning: encoding: %#x\n",
 716                              __func__, encoding & 0x70);
 717                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
 718         }
 719
 720         if (res != 0) {
 721                 if (encoding & DW_EH_PE_indirect) {
 722                         pr_debug("DW_EH_PE_indirect\n");
 723
 724                         if (dw_ptr_size == 4) {
 725                                 res = read_mmap_data_u32(ri, (u32 *)res,
 726                                                          st, &err);
 727                         } else if (dw_ptr_size == 8) {
 728                                 res = read_mmap_data_u64(ri, (u64 *)res,
 729                                                          st, &err);
 730                         } else {
 731                                 pr_err_once("error: wrong dwarf size\n");
 732                                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
 733                         }
 734
 735                         /* we ignore links to unloaded sections */
 736                         if (err)
 737                                 res = 0;
 738                 }
 739         }
 740
 741         *val = res;
 742
 743         return count;
 744 }
 745
 746 static long
 747 dwarf_cfa_exec_insns(struct ex_region_info *ri,
 748                      unsigned char *insn_start,
 749                      unsigned char *insn_end,
 750                      struct dw_cie *cie,
 751                      struct stackframe *sf,
 752                      unsigned long pc,
 753                      int is_eh)
 754 {
 755         unsigned char insn;
 756         unsigned char *c_insn;
 757         unsigned int expr_len, delta, secid;
 758         unsigned long utmp, reg;
 759         long offset, stmp, err = 0;
 760         struct regs_state *rs, *rs_initial, *rs_stack;
 761         struct dwarf_cpu_context *cpu_ctx = this_cpu_ptr(ctx.cpu_ctx);
 762
 763         secid = get_secid_frame(is_eh);
 764
 765         rs = &sf->rs;
 766         rs_initial = &sf->rs_initial;
 767
 768         rs_stack = cpu_ctx->rs_stack;
 769         cpu_ctx->depth = 0;
 770
 771         c_insn = insn_start;
 772
 773         while (c_insn < insn_end && sf->pc <= pc) {
 774                 insn = read_mmap_data_u8(ri, c_insn++,
 775                                          secid, &err);
 776                 if (err)
 777                         return err;
 778
 779                 switch (dw_cfa_opcode(insn)) {
 780                 case DW_CFA_advance_loc:
 781                         delta = dw_cfa_operand(insn);
 782                         delta *= cie->code_align_factor;
 783                         sf->pc += delta;
 784                         pr_debug("DW_CFA_advance_loc: pc: %#lx --> %#lx (delta: %#x)\n",
 785                                 sf->pc - delta, sf->pc, delta);
 786                         continue;
 787
 788                 case DW_CFA_offset:
 789                         reg = dw_cfa_operand(insn);
 790                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 791                                                      secid, &err);
 792                         if (err)
 793                                 return err;
 794
 795                         offset = utmp * cie->data_align_factor;
 796                         set_rule_offset(rs, reg, DW_WHERE_CFAREL, offset);
 797                         pr_debug("DW_CFA_offset: reg: r%lu, offset(addr): %#lx (%ld)\n",
 798                                 reg, offset, offset);
 799                         continue;
 800
 801                 case DW_CFA_restore:
 802                         reg = dw_cfa_operand(insn);
 803
 804                         if (!validate_regnum(rs, reg))
 805                                 break;
 806
 807                         rs->reg[reg] = rs_initial->reg[reg];
 808                         pr_debug("DW_CFA_restore: reg: r%lu\n", reg);
 809                         continue;
 810                 }
 811
 812                 switch (insn) {
 813                 case DW_CFA_nop:
 814                         pr_debug("DW_CFA_nop\n");
 815                         continue;
 816
 817                 case DW_CFA_advance_loc1:
 818                         delta = read_mmap_data_u8(ri, c_insn++,
 819                                                   secid, &err);
 820                         if (err)
 821                                 return err;
 822
 823                         sf->pc += delta * cie->code_align_factor;
 824                         pr_debug("DW_CFA_advance_loc1: pc: %#lx --> %#lx (delta: %#lx)\n",
 825                                 sf->pc - delta * cie->code_align_factor, sf->pc,
 826                                 delta * cie->code_align_factor);
 827                         break;
 828
 829                 case DW_CFA_advance_loc2:
 830                         delta = read_mmap_data_u16(ri, (u16 *)c_insn,
 831                                                    secid, &err);
 832                         if (err)
 833                                 return err;
 834
 835                         c_insn += 2;
 836                         sf->pc += delta * cie->code_align_factor;
 837                         pr_debug("DW_CFA_advance_loc2: pc: %#lx --> %#lx (delta: %#lx)\n",
 838                                 sf->pc - delta * cie->code_align_factor, sf->pc,
 839                                 delta * cie->code_align_factor);
 840                         break;
 841
 842                 case DW_CFA_advance_loc4:
 843                         delta = read_mmap_data_u32(ri, (u32 *)c_insn,
 844                                                    secid, &err);
 845                         if (err)
 846                                 return err;
 847
 848                         c_insn += 4;
 849                         sf->pc += delta * cie->code_align_factor;
 850                         pr_debug("DW_CFA_advance_loc4: pc: %#lx --> %#lx (delta: %#lx)\n",
 851                                 sf->pc - delta * cie->code_align_factor, sf->pc,
 852                                 delta * cie->code_align_factor);
 853                         break;
 854
 855                 case DW_CFA_offset_extended:
 856                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 857                                                      secid, &err);
 858                         if (err)
 859                                 return err;
 860
 861                         reg = utmp;
 862                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 863                                                      secid, &err);
 864                         if (err)
 865                                 return err;
 866
 867                         offset = utmp * cie->data_align_factor;
 868                         pr_debug("DW_CFA_offset_extended: reg: r%lu, offset: %#lx\n",
 869                                  reg, offset);
 870                         break;
 871
 872                 case DW_CFA_restore_extended:
 873                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
 874                                                      secid, &err);
 875                         if (err)
 876                                 return err;
 877
 878                         pr_debug("DW_CFA_restore_extended: reg: r%lu\n", reg);
 879                         break;
 880
 881                 case DW_CFA_undefined:
 882                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
 883                                                      secid, &err);
 884                         if (err)
 885                                 return err;
 886
 887                         set_rule(rs, reg, DW_WHERE_UNDEF, 0);
 888                         pr_debug("DW_CFA_undefined: reg: r%lu\n", reg);
 889                         break;
 890
 891                 case DW_CFA_def_cfa:
 892                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 893                                                      secid, &err);
 894                         if (err)
 895                                 return err;
 896
 897                         rs->cfa_register = utmp;
 898                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 899                                                      secid, &err);
 900                         if (err)
 901                                 return err;
 902
 903                         rs->cfa_offset = utmp;
 904                         pr_debug("DW_CFA_def_cfa: cfa_register: r%u, cfa_offset: %ld (%#lx)\n",
 905                                  rs->cfa_register, rs->cfa_offset,
 906                                  rs->cfa_offset);
 907                         break;
 908
 909                 case DW_CFA_def_cfa_register:
 910                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 911                                                      secid, &err);
 912                         if (err)
 913                                 return err;
 914
 915                         rs->cfa_register = utmp;
 916                         pr_debug("DW_CFA_def_cfa_register: cfa_register: r%u\n",
 917                                rs->cfa_register);
 918                         break;
 919
 920                 case DW_CFA_def_cfa_offset:
 921                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 922                                                      secid, &err);
 923                         if (err)
 924                                 return err;
 925
 926                         rs->cfa_offset = utmp;
 927                         pr_debug("DW_CFA_def_cfa_offset: cfa_offset: %ld (%#lx)\n",
 928                                rs->cfa_offset, rs->cfa_offset);
 929                         break;
 930
 931                 case DW_CFA_def_cfa_expression:
 932                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 933                                                      secid, &err);
 934                         if (err)
 935                                 return err;
 936
 937                         expr_len = utmp;
 938
 939                         rs->cfa_expr = c_insn;
 940                         rs->cfa_expr_len = expr_len;
 941                         rs->cfa_how = DW_CFA_EXP;
 942                         c_insn += expr_len;
 943
 944                         pr_debug("DW_CFA_def_cfa_expression: expr_len: %#x\n",
 945                                  expr_len);
 946                         break;
 947
 948                 case DW_CFA_expression:
 949                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
 950                                                      secid, &err);
 951                         if (err)
 952                                 return err;
 953
 954                         set_rule_exp(rs, reg, DW_WHERE_EXPR, c_insn);
 955
 956                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 957                                                      secid, &err);
 958                         if (err)
 959                                 return err;
 960
 961                         c_insn += utmp;
 962
 963                         pr_debug("DW_CFA_expression: reg: r%lu\n", reg);
 964                         break;
 965
 966                 case DW_CFA_offset_extended_sf:
 967                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
 968                                                      secid, &err);
 969                         if (err)
 970                                 return err;
 971
 972                         c_insn += dwarf_read_sleb128(ri, c_insn, &stmp,
 973                                                      secid, &err);
 974                         if (err)
 975                                 return err;
 976
 977                         offset = stmp * cie->data_align_factor;
 978                         set_rule_offset(rs, reg, DW_WHERE_CFAREL, offset);
 979                         pr_debug("DW_CFA_offset_extended_sf: reg: r%lu, offset: %#lx\n",
 980                                  reg, offset);
 981                         break;
 982
 983                 case DW_CFA_val_offset:
 984                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
 985                                                      secid, &err);
 986                         if (err)
 987                                 return err;
 988
 989                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 990                                                      secid, &err);
 991                         if (err)
 992                                 return err;
 993
 994                         offset = utmp * cie->data_align_factor;
 995                         set_rule_offset(rs, reg, DW_WHERE_VAL_OFFSET, offset);
 996                         pr_debug("DW_CFA_val_offset: reg: r%lu, offset(addr): %#lx\n",
 997                                  reg, offset);
 998                         break;
 999
1000                 case DW_CFA_val_offset_sf:
1001                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
1002                                                      secid, &err);
1003                         if (err)
1004                                 return err;
1005
1006                         c_insn += dwarf_read_sleb128(ri, c_insn, &stmp,
1007                                                      secid, &err);
1008                         if (err)
1009                                 return err;
1010
1011                         offset = stmp * cie->data_align_factor;
1012                         set_rule_offset(rs, reg, DW_WHERE_VAL_OFFSET, offset);
1013                         pr_debug("DW_CFA_val_offset_sf: reg: r%lu, offset(addr): %#lx\n",
1014                                  reg, offset);
1015                         break;
1016
1017                 case DW_CFA_GNU_args_size:
1018                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
1019                                                      secid, &err);
1020                         if (err)
1021                                 return err;
1022
1023                         pr_debug("DW_CFA_GNU_args_size: offset: %#lx\n", utmp);
1024                         break;
1025
1026                 case DW_CFA_GNU_negative_offset_extended:
1027                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
1028                                                      secid, &err);
1029                         if (err)
1030                                 return err;
1031
1032                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
1033                                                      secid, &err);
1034                         if (err)
1035                                 return err;
1036
1037                         offset = utmp * cie->data_align_factor;
1038                         set_rule_offset(rs, reg, DW_WHERE_CFAREL, -offset);
1039                         pr_debug("DW_CFA_GNU_negative_offset_extended: reg: r%lu, offset: %#lx\n",
1040                                  reg, offset);
1041                         break;
1042
1043                 case DW_CFA_remember_state:
1044                         pr_debug("DW_CFA_remember_state\n");
1045
1046                         if (cpu_ctx->depth >= DW_MAX_RS_STACK_DEPTH) {
1047                                 pr_warn_once("error: rs stack was overflowed\n");
1048                                 return 0;
1049                         }
1050
1051                         rs_stack[cpu_ctx->depth++] = *rs;
1052                         break;
1053
1054                 case DW_CFA_restore_state:
1055                         pr_debug("DW_CFA_restore_state\n");
1056
1057                         if (cpu_ctx->depth == 0) {
1058                                 pr_warn_once("error: rs stack error\n");
1059                                 return 0;
1060                         }
1061
1062                         *rs = rs_stack[--cpu_ctx->depth];
1063                         break;
1064
1065                 case DW_CFA_def_cfa_sf:
1066                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
1067                                                      secid, &err);
1068                         if (err)
1069                                 return err;
1070
1071                         c_insn += dwarf_read_sleb128(ri, c_insn, &stmp,
1072                                                      secid, &err);
1073                         if (err)
1074                                 return err;
1075
1076                         rs->cfa_register = utmp;
1077                         rs->cfa_offset = stmp * cie->data_align_factor;
1078                         rs->cfa_how = DW_CFA_REG_OFFSET;
1079
1080                         pr_debug("DW_CFA_def_cfa_sf: cfa_register: r%u, cfa_offset: %ld (%#lx)\n",
1081                                 rs->cfa_register, rs->cfa_offset,
1082                                 rs->cfa_offset);
1083                         break;
1084
1085                 case DW_CFA_def_cfa_offset_sf:
1086                         c_insn += dwarf_read_sleb128(ri, c_insn, &stmp,
1087                                                      secid, &err);
1088                         if (err)
1089                                 return err;
1090
1091                         rs->cfa_offset = stmp * cie->data_align_factor;
1092                         pr_debug("DW_CFA_def_cfa_offset_sf: cfa_offset: %ld (%#lx)\n",
1093                                 rs->cfa_offset, rs->cfa_offset);
1094                         break;
1095
1096                 case DW_CFA_same_value:
1097                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
1098                                                      secid, &err);
1099                         if (err)
1100                                 return err;
1101
1102                         set_rule(rs, reg, DW_WHERE_SAME, 0);
1103                         pr_debug("DW_CFA_same_value: reg: r%lu\n", reg);
1104                         break;
1105
1106                 case DW_CFA_val_expression:
1107                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
1108                                                      secid, &err);
1109                         if (err)
1110                                 return err;
1111
1112                         set_rule_exp(rs, reg, DW_WHERE_VAL_EXPR, c_insn);
1113                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
1114                                                      secid, &err);
1115                         if (err)
1116                                 return err;
1117
1118                         c_insn += utmp;
1119                         pr_debug("DW_CFA_val_expression: reg: r%lu\n", reg);
1120                         break;
1121
1122                 default:
1123                         pr_warn_once("warning: unhandled dwarf instr %#x\n",
1124                                      insn);
1125                         break;
1126                 }
1127         }
1128
1129         return 0;
1130 }
1131
1132 static long
1133 decode_cie_entry(struct ex_region_info *ri,
1134                  struct dw_cie *cie,
1135                  unsigned char *entry,
1136                  size_t length,
1137                  int is_eh)
1138 {
1139         long err;
1140         unsigned long utmp;
1141         unsigned char *p, *end, *aug;
1142         unsigned int secid, cie_id;
1143         unsigned int cie_version, id, len, max_len;
1144
1145         secid = get_secid_frame(is_eh);
1146
1147         p = entry;
1148         end = entry + length;
1149
1150         p += sizeof(u32);
1151
1152         id = read_mmap_data_u32(ri, (u32 *)p, secid, &err);
1153         if (err)
1154                 return err;
1155
1156         p += sizeof(u32);
1157
1158         cie_id = is_eh ? 0 : DW_CIE_ID;
1159         if (id != cie_id) {
1160                 pr_err_once("error: incorrect cie_id");
1161                 return -QUADD_URC_TBL_IS_CORRUPT;
1162         }
1163
1164         cie_version = read_mmap_data_u8(ri, p++, secid, &err);
1165         if (err)
1166                 return err;
1167
1168         if (cie_version != 1 && cie_version != 3) {
1169                 pr_err_once("error: wrong cie_version: %u\n", cie_version);
1170                 return -QUADD_URC_TBL_IS_CORRUPT;
1171         }
1172
1173         if (p >= end)
1174                 return -QUADD_URC_TBL_IS_CORRUPT;
1175
1176         max_len = end - p - 1;
1177         len = strnlen((const char *)p, max_len);
1178         if (len == max_len)
1179                 return -QUADD_URC_TBL_IS_CORRUPT;
1180
1181         cie->aug_string = p;
1182         p += len + 1;
1183
1184         pr_debug("aug_string: %s\n", cie->aug_string);
1185
1186         p += dwarf_read_uleb128(ri, p, &cie->code_align_factor,
1187                                 secid, &err);
1188         if (err)
1189                 return err;
1190
1191         p += dwarf_read_sleb128(ri, p, &cie->data_align_factor,
1192                                 secid, &err);
1193         if (err)
1194                 return err;
1195
1196         if (cie_version == 1) {
1197                 cie->retaddr_reg = read_mmap_data_u8(ri, p++,
1198                                                      secid,
1199                                                      &err);
1200                 if (err)
1201                         return err;
1202         } else {
1203                 p += dwarf_read_uleb128(ri, p, &utmp,
1204                                         secid, &err);
1205                 if (err)
1206                         return err;
1207
1208                 cie->retaddr_reg = utmp;
1209         }
1210
1211         pr_debug("address column: %u\n", cie->retaddr_reg);
1212
1213         aug = cie->aug_string;
1214         cie->z_aug = 0;
1215
1216         cie->initial_insn = NULL;
1217         cie->initial_insn_len = 0;
1218
1219         if (*aug == 'z') {
1220                 p += dwarf_read_uleb128(ri, p, &cie->aug_size,
1221                                         secid, &err);
1222                 if (err)
1223                         return err;
1224
1225                 cie->initial_insn = p + cie->aug_size;
1226                 aug++;
1227
1228                 cie->z_aug = 1;
1229         }
1230
1231         cie->fde_encoding = 0;
1232         cie->lsda_encoding = DW_EH_PE_omit;
1233         cie->personality = NULL;
1234
1235         while (*aug != '\0') {
1236                 if (p >= end)
1237                         return -QUADD_URC_TBL_IS_CORRUPT;
1238
1239                 if (*aug == 'L') {
1240                         cie->lsda_encoding =
1241                                 read_mmap_data_u8(ri, p++,
1242                                                   secid,
1243                                                   &err);
1244                         if (err)
1245                                 return err;
1246
1247                         aug++;
1248                 } else if (*aug == 'R') {
1249                         cie->fde_encoding =
1250                                 read_mmap_data_u8(ri, p++,
1251                                                   secid,
1252                                                   &err);
1253                         if (err)
1254                                 return err;
1255
1256                         aug++;
1257                         pr_debug("fde_encoding: %#x\n", cie->fde_encoding);
1258                 } else if (*aug == 'P') {
1259                         int cnt;
1260                         void *pcrel_base;
1261                         unsigned char handler_encoding;
1262                         unsigned long personality;
1263
1264                         handler_encoding = *p++;
1265
1266                         pcrel_base = (void *)
1267                                 mmap_addr_to_ex_addr((unsigned long)p,
1268                                                      ri,
1269                                                      secid);
1270
1271                         cnt = dwarf_read_encoded_value(ri, p, pcrel_base,
1272                                                        &personality,
1273                                                        handler_encoding,
1274                                                        secid);
1275                         if (cnt < 0) {
1276                                 pr_err_once("%s: error: personality routine\n",
1277                                             __func__);
1278                                 return cnt;
1279                         }
1280                         p += cnt;
1281
1282                         pr_debug("personality: %#lx\n", personality);
1283                         cie->personality = (void *)personality;
1284                         aug++;
1285                 } else if (*aug == 'S') {
1286                         aug++;
1287                         pr_debug("%s: aug: S\n", __func__);
1288                 } else {
1289                         pr_warn_once("%s: warning: unknown aug\n", __func__);
1290                         return -QUADD_URC_UNHANDLED_INSTRUCTION;
1291                 }
1292         }
1293
1294         if (p > end) {
1295                 pr_err_once("%s: error: cie\n", __func__);
1296                 return -QUADD_URC_TBL_IS_CORRUPT;
1297         }
1298
1299         if (p == end)
1300                 return 0;
1301
1302         if (!cie->initial_insn)
1303                 cie->initial_insn = p;
1304
1305         cie->initial_insn_len = end - cie->initial_insn;
1306
1307         return 0;
1308 }
1309
1310 static long
1311 decode_fde_entry(struct ex_region_info *ri,
1312                  struct dw_fde *fde,
1313                  unsigned char *entry,
1314                  size_t length,
1315                  int is_eh)
1316 {
1317         int count, secid;
1318         long err = 0;
1319         unsigned long utmp;
1320         unsigned char *p, *end, *pcrel_base;
1321         struct dw_cie *cie = fde->cie;
1322
1323         secid = get_secid_frame(is_eh);
1324
1325         p = entry;
1326         end = entry + length;
1327
1328         p += sizeof(u32);
1329         p += sizeof(u32);
1330
1331         pcrel_base = (unsigned char *)
1332                 mmap_addr_to_ex_addr((unsigned long)p, ri, secid);
1333
1334         count = dwarf_read_encoded_value(ri, p, pcrel_base,
1335                                          &fde->initial_location,
1336                                          cie->fde_encoding,
1337                                          secid);
1338         if (count < 0)
1339                 return count;
1340
1341         p += count;
1342
1343         fde->address_range = read_mmap_data_u32(ri, (u32 *)p,
1344                                                 secid, &err);
1345         if (err)
1346                 return err;
1347
1348         p += sizeof(u32);
1349
1350         if (fde->initial_location < ri->vm_start)
1351                 fde->initial_location += ri->vm_start;
1352
1353         pr_debug("pcrel_base: %p\n", pcrel_base);
1354         pr_debug("init location: %#lx\n", fde->initial_location);
1355         pr_debug("address_range: %#lx\n", fde->address_range);
1356
1357         if (cie->z_aug) {
1358                 p += dwarf_read_uleb128(ri, p, &utmp,
1359                                         secid, &err);
1360                 if (err)
1361                         return err;
1362
1363                 p += utmp;
1364         }
1365
1366         if (p > end) {
1367                 pr_err_once("%s: error: incorrect fde\n", __func__);
1368                 return -QUADD_URC_TBL_IS_CORRUPT;
1369         }
1370
1371         fde->insn_length = end - p;
1372
1373         if (fde->insn_length > 0)
1374                 fde->instructions = p;
1375         else
1376                 fde->instructions = NULL;
1377
1378         return 0;
1379 }
1380
1381 static const struct dw_fde_table *
1382 dwarf_bst_find_idx(unsigned long data_base,
1383                    struct dw_fde_table *fde_table,
1384                    unsigned long length,
1385                    unsigned long addr)
1386 {
1387         unsigned long initial_loc;
1388         struct dw_fde_table *start, *stop;
1389         struct dw_fde_table *mid = NULL;
1390
1391         if (unlikely(!length))
1392                 return NULL;
1393
1394         start = fde_table;
1395         stop = start + length - 1;
1396
1397         initial_loc = dw_bst_get_initial_loc(start, data_base);
1398         if (addr < initial_loc)
1399                 return NULL;
1400
1401         initial_loc = dw_bst_get_initial_loc(stop, data_base);
1402         if (addr >= initial_loc)
1403                 return NULL;
1404
1405         while (start < stop - 1) {
1406                 mid = start + ((stop - start) >> 1);
1407
1408                 initial_loc = dw_bst_get_initial_loc(mid, data_base);
1409
1410                 if (addr < initial_loc)
1411                         stop = mid;
1412                 else
1413                         start = mid;
1414         }
1415
1416         return start;
1417 }
1418
1419 static struct dw_fde_table *
1420 dwarf_get_bs_table(struct ex_region_info *ri,
1421                    void *data,
1422                    unsigned long length,
1423                    unsigned long data_base,
1424                    unsigned long *nr_entries,
1425                    int is_eh)
1426 {
1427         int count, secid_hdr;
1428         unsigned char *p, *end;
1429         struct dw_fde_table *bst;
1430         unsigned long fde_count, frame_ptr;
1431         struct dw_eh_frame_hdr *hdr = data;
1432
1433         if (length <= sizeof(*hdr))
1434                 return NULL;
1435
1436         end = data + length;
1437
1438         pr_debug("hdr: %p\n", hdr);
1439
1440         secid_hdr = get_secid_frame_hdr(is_eh);
1441
1442         if (hdr->version != 1) {
1443                 pr_warn_once("warning: unknown eh hdr format\n");
1444                 return NULL;
1445         }
1446         p = (unsigned char *)(hdr + 1);
1447
1448         if (hdr->eh_frame_ptr_enc != DW_EH_PE_omit) {
1449                 count = dwarf_read_encoded_value(ri, p, (void *)data_base,
1450                                                  &frame_ptr,
1451                                                  hdr->eh_frame_ptr_enc,
1452                                                  secid_hdr);
1453                 if (count < 0)
1454                         return NULL;
1455
1456                 p += count;
1457         }
1458
1459         if (hdr->fde_count_enc == DW_EH_PE_omit)
1460                 return NULL;
1461
1462         count = dwarf_read_encoded_value(ri, p, (void *)data_base,
1463                                          &fde_count, hdr->fde_count_enc,
1464                                          secid_hdr);
1465         if (count < 0)
1466                 return NULL;
1467
1468         p += count;
1469
1470         if (p >= end)
1471                 return NULL;
1472
1473         if (fde_count * sizeof(*bst) !=  end - p)
1474                 return NULL;
1475
1476         if (hdr->table_enc != (DW_EH_PE_datarel | DW_EH_PE_sdata4)) {
1477                 pr_warn_once("warning: unknown eh hdr format\n");
1478                 return NULL;
1479         }
1480
1481         bst = (struct dw_fde_table *)p;
1482         *nr_entries = fde_count;
1483
1484         pr_debug("bst: %lu fde entries\n", fde_count);
1485
1486         return bst;
1487 }
1488
1489 static long
1490 dwarf_decode_fde_cie(struct ex_region_info *ri,
1491                      unsigned char *fde_p,
1492                      struct dw_cie *cie,
1493                      struct dw_fde *fde,
1494                      int is_eh)
1495 {
1496         u32 *p;
1497         int secid;
1498         long err;
1499         unsigned char *cie_p;
1500         unsigned long cie_pointer, length;
1501         unsigned char *frame_start;
1502         unsigned long frame_len, addr;
1503         struct extab_info *ti;
1504
1505         secid = get_secid_frame(is_eh);
1506         ti = &ri->ex_sec[secid];
1507
1508         addr = ti->addr;
1509
1510         frame_start = (unsigned char *)
1511                 ex_addr_to_mmap_addr(addr, ri, secid);
1512
1513         frame_len = ti->length;
1514
1515         pr_debug("frame: %p - %p\n",
1516                  frame_start, frame_start + frame_len);
1517
1518         p = (u32 *)fde_p;
1519
1520         length = read_mmap_data_u32(ri, p++, secid, &err);
1521         if (err)
1522                 return err;
1523
1524         if (length == 0xffffffff) {
1525                 pr_warn_once("warning: 64-bit frame is not supported\n");
1526                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
1527         }
1528
1529         fde->offset = fde_p - frame_start;
1530         fde->length = length + sizeof(u32);
1531
1532         pr_debug("FDE: fde_p: %p, offset: %#lx, len: %#lx\n",
1533                  fde_p, fde->offset, fde->length);
1534
1535         cie_pointer = read_mmap_data_u32(ri, p, secid, &err);
1536         if (err)
1537                 return err;
1538
1539         fde->cie_pointer = cie_pointer;
1540
1541         cie_p = is_eh ? (unsigned char *)p - cie_pointer :
1542                 frame_start + cie_pointer;
1543
1544         length = read_mmap_data_u32(ri, (u32 *)cie_p,
1545                                     secid, &err);
1546         if (err)
1547                 return err;
1548
1549         if (length == 0xffffffff) {
1550                 pr_warn_once("warning: 64-bit frame is not supported\n");
1551                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
1552         }
1553
1554         cie->offset = cie_p - frame_start;
1555         cie->length = length + sizeof(u32);
1556
1557         pr_debug("CIE: cie_p: %p, offset: %#lx, len: %#lx\n",
1558                  cie_p, cie->offset, cie->length);
1559
1560         err = decode_cie_entry(ri, cie, cie_p, cie->length, is_eh);
1561         if (err < 0)
1562                 return err;
1563
1564         fde->cie = cie;
1565
1566         err = decode_fde_entry(ri, fde, fde_p, fde->length, is_eh);
1567         if (err < 0)
1568                 return err;
1569
1570         return 0;
1571 }
1572
1573 static void *
1574 dwarf_find_fde(struct ex_region_info *ri,
1575                void *data,
1576                unsigned long length,
1577                unsigned long pc,
1578                int is_eh)
1579 {
1580         long err;
1581         int secid, secid_hdr;
1582         const struct dw_fde_table *fi;
1583         unsigned long fde_count = 0, data_base;
1584         unsigned long fde_addr, init_loc;
1585         struct dw_fde_table *bst;
1586         struct extab_info *ti;
1587
1588         secid = get_secid_frame(is_eh);
1589         secid_hdr = get_secid_frame_hdr(is_eh);
1590
1591         ti = &ri->ex_sec[secid_hdr];
1592         data_base = ti->addr;
1593
1594         bst = dwarf_get_bs_table(ri, data, length, data_base,
1595                                  &fde_count, is_eh);
1596         if (!bst || fde_count == 0) {
1597                 pr_warn_once("warning: bs_table\n");
1598                 return NULL;
1599         }
1600
1601         fi = &bst[fde_count - 1];
1602         init_loc = dw_bst_get_initial_loc(fi, data_base);
1603
1604         pr_debug("pc: %#lx, last bst init_loc: %#lx", pc, init_loc);
1605
1606         if (pc >= init_loc) {
1607                 unsigned long start, end;
1608                 struct extab_info *ti = &ri->ex_sec[secid];
1609
1610                 fde_addr = dw_bst_get_fde_addr(fi, data_base);
1611                 fde_addr = ex_addr_to_mmap_addr(fde_addr, ri,
1612                                                 secid);
1613
1614                 if (pc == init_loc)
1615                         return (void *)fde_addr;
1616
1617                 if (ti->tf_end > 0) {
1618                         start = ti->tf_start;
1619                         end = ti->tf_end;
1620                 } else {
1621                         struct dw_cie cie;
1622                         struct dw_fde fde;
1623
1624                         err = dwarf_decode_fde_cie(ri, (void *)fde_addr,
1625                                                    &cie, &fde, is_eh);
1626                         if (err < 0)
1627                                 return NULL;
1628
1629                         start = fde.initial_location;
1630                         end = start + fde.address_range;
1631
1632                         quadd_unwind_set_tail_info(ri->vm_start, secid,
1633                                                    start, end);
1634                 }
1635
1636                 pr_debug("pc: %#lx, last bst entry: %#lx - %#lx",
1637                          pc, start, end);
1638
1639                 return (pc >= start && pc < end) ?
1640                        (void *)fde_addr : NULL;
1641         }
1642
1643         fi = dwarf_bst_find_idx(data_base, bst, fde_count, pc);
1644         if (!fi)
1645                 return NULL;
1646
1647         fde_addr = dw_bst_get_fde_addr(fi, data_base);
1648         fde_addr = ex_addr_to_mmap_addr(fde_addr, ri, secid);
1649
1650         return (void *)fde_addr;
1651 }
1652
1653 static int
1654 __is_fde_entry_exist(struct ex_region_info *ri, unsigned long addr, int is_eh)
1655 {
1656         int secid_hdr;
1657         unsigned char *fde_p;
1658         struct extab_info *ti;
1659         unsigned char *hdr_start;
1660         unsigned long hdr_len, a;
1661
1662         secid_hdr = get_secid_frame_hdr(is_eh);
1663
1664         ti = &ri->ex_sec[secid_hdr];
1665
1666         a = ti->addr;
1667
1668         hdr_start = (unsigned char *)
1669                 ex_addr_to_mmap_addr(a, ri, secid_hdr);
1670
1671         hdr_len = ti->length;
1672
1673         fde_p = dwarf_find_fde(ri, hdr_start, hdr_len, addr, is_eh);
1674
1675         return fde_p ? 1 : 0;
1676 }
1677
1678 static int
1679 is_fde_entry_exist(struct ex_region_info *ri,
1680                    unsigned long addr,
1681                    int *is_eh,
1682                    int *is_debug)
1683 {
1684         *is_eh = 0;
1685         *is_debug = 0;
1686
1687         if (is_frame_present(ri, 1)) {
1688                 if (__is_fde_entry_exist(ri, addr, 1))
1689                         *is_eh = 1;
1690         }
1691
1692         if (is_frame_present(ri, 0)) {
1693                 if (__is_fde_entry_exist(ri, addr, 0))
1694                         *is_debug = 1;
1695         }
1696
1697         return (*is_eh || *is_debug) ? 1 : 0;
1698 }
1699
1700 static long
1701 dwarf_decode(struct ex_region_info *ri,
1702              struct dw_cie *cie,
1703              struct dw_fde *fde,
1704              unsigned long pc,
1705              int is_eh)
1706 {
1707         long err;
1708         int secid_hdr;
1709         unsigned char *fde_p;
1710         unsigned char *hdr_start;
1711         unsigned long hdr_len, addr;
1712         struct extab_info *ti;
1713
1714         secid_hdr = get_secid_frame_hdr(is_eh);
1715         ti = &ri->ex_sec[secid_hdr];
1716
1717         addr = ti->addr;
1718
1719         hdr_start = (unsigned char *)
1720                 ex_addr_to_mmap_addr(addr, ri, secid_hdr);
1721
1722         hdr_len = ti->length;
1723
1724         pr_debug("eh frame hdr: %p - %p\n",
1725                  hdr_start, hdr_start + hdr_len);
1726
1727         fde_p = dwarf_find_fde(ri, hdr_start, hdr_len, pc, is_eh);
1728         if (!fde_p)
1729                 return -QUADD_URC_IDX_NOT_FOUND;
1730
1731         err = dwarf_decode_fde_cie(ri, fde_p, cie, fde, is_eh);
1732         if (err < 0)
1733                 return err;
1734
1735         if (pc < fde->initial_location ||
1736             pc >= fde->initial_location + fde->address_range) {
1737                 pr_debug("pc is not in range: %#lx - %#lx\n",
1738                          fde->initial_location,
1739                          fde->initial_location + fde->address_range);
1740                 return -QUADD_URC_IDX_NOT_FOUND;
1741         }
1742
1743         return 0;
1744 }
1745
1746 static long def_cfa(struct stackframe *sf, struct regs_state *rs)
1747 {
1748         int reg = rs->cfa_register;
1749
1750         if (reg >= 0) {
1751                 if (reg >= QUADD_NUM_REGS)
1752                         return -QUADD_URC_TBL_IS_CORRUPT;
1753
1754                 pr_debug("r%d --> cfa (%#lx)\n", reg, sf->cfa);
1755                 sf->cfa = sf->vregs[reg];
1756         }
1757
1758         sf->cfa += rs->cfa_offset;
1759         pr_debug("cfa += %#lx (%#lx)\n", rs->cfa_offset, sf->cfa);
1760
1761         return 0;
1762 }
1763
1764 static long
1765 unwind_frame(struct ex_region_info *ri,
1766              struct stackframe *sf,
1767              struct vm_area_struct *vma_sp,
1768              int is_eh)
1769 {
1770         int i;
1771         long err;
1772         unsigned char *insn_end;
1773         unsigned long addr, return_addr, val, user_reg_size;
1774         struct dw_fde fde;
1775         struct dw_cie cie;
1776         unsigned long pc = sf->pc;
1777         struct regs_state *rs, *rs_initial;
1778         int mode = sf->mode;
1779
1780         err = dwarf_decode(ri, &cie, &fde, pc, is_eh);
1781         if (err < 0)
1782                 return err;
1783
1784         sf->pc = fde.initial_location;
1785
1786         rs = &sf->rs;
1787         rs_initial = &sf->rs_initial;
1788
1789         rs->cfa_register = -1;
1790         rs_initial->cfa_register = -1;
1791
1792         set_rule(rs, regnum_lr(mode), DW_WHERE_UNDEF, 0);
1793
1794         if (cie.initial_insn) {
1795                 insn_end = cie.initial_insn + cie.initial_insn_len;
1796                 err = dwarf_cfa_exec_insns(ri, cie.initial_insn,
1797                                            insn_end, &cie, sf, pc, is_eh);
1798                 if (err)
1799                         return err;
1800         }
1801
1802         memcpy(rs_initial, rs, sizeof(*rs));
1803
1804         if (fde.instructions) {
1805                 insn_end = fde.instructions + fde.insn_length;
1806                 err = dwarf_cfa_exec_insns(ri, fde.instructions,
1807                                            insn_end, fde.cie, sf, pc, is_eh);
1808                 if (err)
1809                         return err;
1810         }
1811
1812         pr_debug("mode: %s\n", (mode == DW_MODE_ARM32) ? "arm32" : "arm64");
1813         pr_debug("initial cfa: %#lx\n", sf->cfa);
1814
1815         user_reg_size = get_user_reg_size(mode);
1816
1817         err = def_cfa(sf, rs);
1818         if (err < 0)
1819                 return err;
1820
1821         pr_debug("pc: %#lx, lr: %#lx\n", sf->pc, sf->vregs[regnum_lr(mode)]);
1822
1823         pr_debug("sp: %#lx, fp: %#lx, fp_thumb: %#lx\n",
1824                  sf->vregs[regnum_sp(mode)],
1825                  sf->vregs[regnum_fp(mode)],
1826                  sf->vregs[ARM32_FP_THUMB]);
1827
1828         pr_debug("lr rule: %#lx/%ld (where: %u)\n",
1829                  rs->reg[regnum_lr(mode)].loc.reg,
1830                  rs->reg[regnum_lr(mode)].loc.offset,
1831                  rs->reg[regnum_lr(mode)].where);
1832
1833         pr_debug("fp rule: %#lx/%ld (where: %u)\n",
1834                  rs->reg[regnum_fp(mode)].loc.reg,
1835                  rs->reg[regnum_fp(mode)].loc.offset,
1836                  rs->reg[regnum_fp(mode)].where);
1837
1838         pr_debug("fp_thumb rule: %#lx/%ld (where: %u)\n",
1839                  rs->reg[ARM32_FP_THUMB].loc.reg,
1840                  rs->reg[ARM32_FP_THUMB].loc.offset,
1841                  rs->reg[ARM32_FP_THUMB].where);
1842
1843         pr_debug("cfa_offset: %ld (%#lx)\n",
1844                  rs->cfa_offset, rs->cfa_offset);
1845         pr_debug("cfa_register: %u\n", rs->cfa_register);
1846         pr_debug("new cfa: %#lx\n", sf->cfa);
1847
1848         for (i = 0; i < QUADD_NUM_REGS; i++) {
1849                 switch (rs->reg[i].where) {
1850                 case DW_WHERE_UNDEF:
1851                         break;
1852
1853                 case DW_WHERE_SAME:
1854                         break;
1855
1856                 case DW_WHERE_CFAREL:
1857                         addr = sf->cfa + rs->reg[i].loc.offset;
1858
1859                         if (!validate_stack_addr(addr, vma_sp, user_reg_size))
1860                                 return -QUADD_URC_SP_INCORRECT;
1861
1862                         if (mode == DW_MODE_ARM32)
1863                                 err = read_user_data((u32 __user *)addr, val);
1864                         else
1865                                 err = read_user_data((unsigned long __user *)
1866                                                      addr, val);
1867
1868                         if (err < 0)
1869                                 return err;
1870
1871                         sf->vregs[i] = val;
1872                         pr_debug("[r%d] DW_WHERE_CFAREL: new val: %#lx\n",
1873                                  i, val);
1874
1875                         break;
1876
1877                 default:
1878                         pr_err_once("[r%d] error: unsupported rule\n",
1879                                     rs->reg[i].where);
1880                         break;
1881                 }
1882         }
1883
1884         return_addr = sf->vregs[regnum_lr(mode)];
1885         pr_debug("return_addr: %#lx\n", return_addr);
1886
1887         if (!validate_pc_addr(return_addr, user_reg_size))
1888                 return -QUADD_URC_PC_INCORRECT;
1889
1890         sf->pc = return_addr;
1891         sf->vregs[regnum_sp(mode)] = sf->cfa;
1892
1893         return 0;
1894 }
1895
1896 static void
1897 unwind_backtrace(struct quadd_callchain *cc,
1898                  struct ex_region_info *ri,
1899                  struct stackframe *sf,
1900                  struct vm_area_struct *vma_sp,
1901                  struct task_struct *task)
1902 {
1903         unsigned long user_reg_size;
1904         struct ex_region_info ri_new;
1905         unsigned int unw_type;
1906         int is_eh = 1, mode = sf->mode;
1907
1908         cc->urc_dwarf = QUADD_URC_FAILURE;
1909         user_reg_size = get_user_reg_size(mode);
1910
1911         while (1) {
1912                 long sp, err;
1913                 int nr_added;
1914                 int __is_eh, __is_debug;
1915                 struct vm_area_struct *vma_pc;
1916                 unsigned long addr, where = sf->pc;
1917                 struct mm_struct *mm = task->mm;
1918
1919                 if (!mm)
1920                         break;
1921
1922                 sp = sf->vregs[regnum_sp(mode)];
1923
1924                 if (!validate_stack_addr(sp, vma_sp, user_reg_size)) {
1925                         cc->urc_dwarf = QUADD_URC_SP_INCORRECT;
1926                         break;
1927                 }
1928
1929                 vma_pc = find_vma(mm, sf->pc);
1930                 if (!vma_pc)
1931                         break;
1932
1933                 addr = ri->vm_start;
1934
1935                 if (!is_vma_addr(addr, vma_pc, user_reg_size)) {
1936                         err = quadd_get_dw_frames(vma_pc->vm_start, &ri_new);
1937                         if (err) {
1938                                 cc->urc_dwarf = QUADD_URC_TBL_NOT_EXIST;
1939                                 break;
1940                         }
1941
1942                         pr_debug("ri: %#lx ---> %#lx",
1943                                  ri->vm_start, ri_new.vm_start);
1944
1945                         ri = &ri_new;
1946                 }
1947
1948                 if (!is_fde_entry_exist(ri, sf->pc, &__is_eh, &__is_debug)) {
1949                         pr_debug("eh/debug fde entries are not existed\n");
1950                         cc->urc_dwarf = QUADD_URC_IDX_NOT_FOUND;
1951                         break;
1952                 }
1953                 pr_debug("is_eh: %d, is_debug: %d\n", __is_eh, __is_debug);
1954
1955                 if (is_eh) {
1956                         if (!__is_eh)
1957                                 is_eh = 0;
1958                 } else {
1959                         if (!__is_debug)
1960                                 is_eh = 1;
1961                 }
1962
1963                 err = unwind_frame(ri, sf, vma_sp, is_eh);
1964                 if (err < 0) {
1965                         if (__is_eh && __is_debug) {
1966                                 is_eh ^= 1;
1967
1968                                 err = unwind_frame(ri, sf, vma_sp, is_eh);
1969                                 if (err < 0) {
1970                                         cc->urc_dwarf = -err;
1971                                         break;
1972                                 }
1973                         } else {
1974                                 cc->urc_dwarf = -err;
1975                                 break;
1976                         }
1977                 }
1978
1979                 unw_type = is_eh ? QUADD_UNW_TYPE_DWARF_EH :
1980                                    QUADD_UNW_TYPE_DWARF_DF;
1981
1982                 pr_debug("[%s]: function at [<%08lx>] from [<%08lx>]\n",
1983                          is_eh ? "eh" : "debug", where, sf->pc);
1984
1985                 cc->curr_sp = sf->vregs[regnum_sp(mode)];
1986
1987                 cc->curr_fp = sf->vregs[regnum_fp(mode)];
1988                 if (mode == DW_MODE_ARM32)
1989                         cc->curr_fp_thumb = sf->vregs[ARM32_FP_THUMB];
1990
1991                 cc->curr_pc = sf->pc;
1992
1993                 nr_added = quadd_callchain_store(cc, sf->pc, unw_type);
1994                 if (nr_added == 0)
1995                         break;
1996         }
1997 }
1998
1999 int
2000 quadd_is_ex_entry_exist_dwarf(struct pt_regs *regs,
2001                               unsigned long addr,
2002                               struct task_struct *task)
2003 {
2004         long err;
2005         int is_eh, is_debug;
2006         struct ex_region_info ri;
2007         struct vm_area_struct *vma;
2008         struct mm_struct *mm = task->mm;
2009
2010         if (!regs || !mm)
2011                 return 0;
2012
2013         vma = find_vma(mm, addr);
2014         if (!vma)
2015                 return 0;
2016
2017         err = quadd_get_dw_frames(vma->vm_start, &ri);
2018         if (err)
2019                 return 0;
2020
2021         return is_fde_entry_exist(&ri, addr, &is_eh, &is_debug);
2022 }
2023
2024 unsigned int
2025 quadd_get_user_cc_dwarf(struct pt_regs *regs,
2026                         struct quadd_callchain *cc,
2027                         struct task_struct *task)
2028 {
2029         long err;
2030         int i, mode, nr_prev = cc->nr;
2031         unsigned long ip, lr, sp, fp, fp_thumb;
2032         struct vm_area_struct *vma, *vma_sp;
2033         struct mm_struct *mm = task->mm;
2034         struct ex_region_info ri;
2035         struct stackframe sf;
2036         struct dwarf_cpu_context *cpu_ctx = this_cpu_ptr(ctx.cpu_ctx);
2037
2038         if (!regs || !mm)
2039                 return 0;
2040
2041         if (cc->urc_dwarf == QUADD_URC_LEVEL_TOO_DEEP)
2042                 return nr_prev;
2043
2044         cc->urc_dwarf = QUADD_URC_FAILURE;
2045
2046         if (nr_prev > 0) {
2047                 ip = cc->curr_pc;
2048                 sp = cc->curr_sp;
2049                 fp = cc->curr_fp;
2050                 fp_thumb = cc->curr_fp_thumb;
2051                 lr = 0;
2052         } else {
2053                 ip = instruction_pointer(regs);
2054                 lr = quadd_user_link_register(regs);
2055                 sp = quadd_user_stack_pointer(regs);
2056
2057 #ifdef CONFIG_ARM64
2058                 if (compat_user_mode(regs)) {
2059                         fp = regs->compat_usr(11);
2060                         fp_thumb = regs->compat_usr(7);
2061                 } else {
2062                         fp = regs->regs[29];
2063                         fp_thumb = 0;
2064                 }
2065 #else
2066                 fp = regs->ARM_fp;
2067                 fp_thumb = regs->ARM_r7;
2068 #endif
2069         }
2070
2071 #ifdef CONFIG_ARM64
2072         if (compat_user_mode(regs))
2073                 mode = DW_MODE_ARM32;
2074         else
2075                 mode = DW_MODE_ARM64;
2076 #else
2077         mode = DW_MODE_ARM32;
2078 #endif
2079
2080         pr_debug("%s: pc: %#lx, lr: %#lx\n", __func__, ip, lr);
2081         pr_debug("%s: sp: %#lx, fp: %#lx, fp_thumb: %#lx\n",
2082                  __func__, sp, fp, fp_thumb);
2083
2084         sf.vregs[regnum_lr(mode)] = lr;
2085         sf.pc = ip;
2086
2087         sf.vregs[regnum_sp(mode)] = sp;
2088         sf.vregs[regnum_fp(mode)] = fp;
2089
2090         if (mode == DW_MODE_ARM32)
2091                 sf.vregs[ARM32_FP_THUMB] = fp_thumb;
2092
2093         cpu_ctx->dw_ptr_size = (mode == DW_MODE_ARM32) ?
2094                                 sizeof(u32) : sizeof(u64);
2095
2096         sf.mode = mode;
2097         sf.cfa = 0;
2098
2099         for (i = 0; i < QUADD_NUM_REGS; i++)
2100                 set_rule(&sf.rs, i, DW_WHERE_UNDEF, 0);
2101
2102         vma = find_vma(mm, ip);
2103         if (!vma)
2104                 return 0;
2105
2106         vma_sp = find_vma(mm, sp);
2107         if (!vma_sp)
2108                 return 0;
2109
2110         err = quadd_get_dw_frames(vma->vm_start, &ri);
2111         if (err) {
2112                 cc->urc_dwarf = QUADD_URC_TBL_NOT_EXIST;
2113                 return 0;
2114         }
2115
2116         unwind_backtrace(cc, &ri, &sf, vma_sp, task);
2117
2118         pr_debug("%s: mode: %s, cc->nr: %d --> %d\n", __func__,
2119                  (mode == DW_MODE_ARM32) ? "arm32" : "arm64",
2120                  nr_prev, cc->nr);
2121
2122         return cc->nr;
2123 }
2124
2125 int quadd_dwarf_unwind_start(void)
2126 {
2127         if (!atomic_cmpxchg(&ctx.started, 0, 1)) {
2128                 ctx.cpu_ctx = alloc_percpu(struct dwarf_cpu_context);
2129                 if (!ctx.cpu_ctx) {
2130                         atomic_set(&ctx.started, 0);
2131                         return -ENOMEM;
2132                 }
2133         }
2134
2135         return 0;
2136 }
2137
2138 void quadd_dwarf_unwind_stop(void)
2139 {
2140         if (atomic_cmpxchg(&ctx.started, 1, 0))
2141                 free_percpu(ctx.cpu_ctx);
2142 }
2143
2144 int quadd_dwarf_unwind_init(void)
2145 {
2146         atomic_set(&ctx.started, 0);
2147         return 0;
2148 }