drivers/misc/tegra-profiler/dwarf_unwind.c

   1 /*
   2  * drivers/misc/tegra-profiler/dwarf_unwind.c
   3  *
   4  * Copyright (c) 2015, NVIDIA CORPORATION.  All rights reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  */
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/mm.h>
  20 #include <linux/sched.h>
  21 #include <linux/slab.h>
  22 #include <linux/uaccess.h>
  23 #include <linux/err.h>
  24
  25 #include <asm/unaligned.h>
  26
  27 #include <linux/tegra_profiler.h>
  28
  29 #include "comm.h"
  30 #include "backtrace.h"
  31 #include "eh_unwind.h"
  32 #include "dwarf.h"
  33 #include "dwarf_unwind.h"
  34
  35 enum {
  36         DW_WHERE_UNDEF,         /* register isn't saved at all */
  37         DW_WHERE_SAME,          /* register has same value as in prev. frame */
  38         DW_WHERE_CFAREL,        /* register saved at CFA-relative address */
  39         DW_WHERE_REG,           /* register saved in another register */
  40         DW_WHERE_EXPR,          /* register saved */
  41         DW_WHERE_VAL_OFFSET,    /* value offset */
  42         DW_WHERE_VAL_EXPR,      /* register has computed value */
  43 };
  44
  45 #define QUADD_AARCH64_REGISTERS 32
  46 #define QUADD_AARCH32_REGISTERS 16
  47
  48 #define QUADD_NUM_REGS QUADD_AARCH64_REGISTERS
  49
  50 enum regs32 {
  51         ARM32_FP_THUMB = 7,
  52         ARM32_FP = 11,
  53
  54         ARM32_SP = 13,
  55         ARM32_LR = 14,
  56         ARM32_PC = 15
  57 };
  58
  59 enum regs64 {
  60         ARM64_FP = 29,
  61         ARM64_LR = 30,
  62         ARM64_SP = 31,
  63 };
  64
  65 enum {
  66         DW_MODE_ARM32,
  67         DW_MODE_ARM64,
  68 };
  69
  70 union dw_loc {
  71         unsigned long reg;
  72         long offset;
  73         const unsigned char *exp;
  74 };
  75
  76 struct reg_info {
  77         int where;
  78         union dw_loc loc;
  79 };
  80
  81 enum {
  82         DW_CFA_UNSET,
  83         DW_CFA_REG_OFFSET,
  84         DW_CFA_EXP,
  85 };
  86
  87 struct dw_eh_frame_hdr {
  88         unsigned char version;
  89         unsigned char eh_frame_ptr_enc;
  90         unsigned char fde_count_enc;
  91         unsigned char table_enc;
  92 };
  93
  94 struct dw_fde_table {
  95         s32 initial_loc;
  96         s32 fde;
  97 };
  98
  99 struct regs_state {
 100         struct reg_info reg[QUADD_NUM_REGS];
 101
 102         long cfa_offset;
 103         int cfa_register;
 104
 105         unsigned char *cfa_expr;
 106         unsigned int cfa_expr_len;
 107
 108         int cfa_how;
 109 };
 110
 111 #define DW_MAX_RS_STACK_DEPTH   8
 112
 113 struct dwarf_cpu_context {
 114         struct regs_state rs_stack[DW_MAX_RS_STACK_DEPTH];
 115         int depth;
 116
 117         int dw_word_size;
 118 };
 119
 120 struct quadd_dwarf_context {
 121         struct dwarf_cpu_context __percpu *cpu_ctx;
 122         atomic_t started;
 123 };
 124
 125 struct stackframe {
 126         unsigned long pc;
 127         unsigned long vregs[QUADD_NUM_REGS];
 128
 129         struct regs_state rs;
 130         struct regs_state rs_initial;
 131
 132         unsigned long cfa;
 133
 134         int mode;
 135 };
 136
 137 struct dw_cie {
 138         unsigned long offset;
 139         unsigned long length;
 140
 141         unsigned char *aug_string;
 142         unsigned long aug_size;
 143
 144         unsigned char fde_encoding;
 145         unsigned char lsda_encoding;
 146
 147         unsigned long code_align_factor;
 148         long data_align_factor;
 149
 150         unsigned int initial_insn_len;
 151         unsigned char *initial_insn;
 152
 153         int z_aug;
 154         unsigned int retaddr_reg;
 155         void *personality;
 156
 157         unsigned char *data;
 158 };
 159
 160 struct dw_fde {
 161         unsigned long offset;
 162         unsigned long length;
 163
 164         unsigned long cie_pointer;
 165         struct dw_cie *cie;
 166
 167         unsigned long initial_location;
 168         unsigned long address_range;
 169
 170         unsigned int insn_length;
 171         unsigned char *instructions;
 172
 173         unsigned char *data;
 174 };
 175
 176
 177 struct eh_sec_data {
 178         size_t length;
 179         unsigned char *data;
 180 };
 181
 182 #define read_user_data(addr, retval)                            \
 183 ({                                                              \
 184         long ret;                                               \
 185                                                                 \
 186         pagefault_disable();                                    \
 187         ret = __get_user(retval, addr);                         \
 188         pagefault_enable();                                     \
 189                                                                 \
 190         if (ret) {                                              \
 191                 pr_debug("%s: failed for address: %p\n",        \
 192                          __func__, addr);                       \
 193                 ret = -QUADD_URC_EACCESS;                       \
 194         }                                                       \
 195                                                                 \
 196         ret;                                                    \
 197 })
 198
 199 static struct quadd_dwarf_context ctx;
 200
 201 static inline int regnum_sp(int mode)
 202 {
 203         return (mode == DW_MODE_ARM32) ?
 204                 ARM32_SP : ARM64_SP;
 205 }
 206
 207 static inline int regnum_fp(int mode)
 208 {
 209         return (mode == DW_MODE_ARM32) ?
 210                 ARM32_FP : ARM64_FP;
 211 }
 212
 213 static inline int regnum_lr(int mode)
 214 {
 215         return (mode == DW_MODE_ARM32) ?
 216                 ARM32_LR : ARM64_LR;
 217 }
 218
 219 static inline unsigned long
 220 get_user_reg_size(int mode)
 221 {
 222         return (mode == DW_MODE_ARM32) ?
 223                 sizeof(u32) : sizeof(u64);
 224 }
 225
 226 static inline int
 227 validate_addr(struct ex_region_info *ri,
 228               unsigned long addr,
 229               unsigned long nbytes,
 230               int st)
 231 {
 232         struct extab_info *ti;
 233         struct quadd_mmap_area *mmap;
 234         unsigned long start, end;
 235
 236         mmap = ri->mmap;
 237
 238         ti = &ri->ex_sec[st];
 239
 240         start = (unsigned long)mmap->data + ti->mmap_offset;
 241         end = start + ti->length;
 242
 243         if (unlikely(addr < start || addr > end - nbytes)) {
 244                 pr_err_once("%s: error: addr: %#lx, len: %ld, data: %#lx-%#lx\n",
 245                             __func__, addr, nbytes, start, end);
 246                 return 0;
 247         }
 248
 249         return 1;
 250 }
 251
 252 static inline u8
 253 read_mmap_data_u8(struct ex_region_info *ri,
 254                   const u8 *addr, int st, long *err)
 255 {
 256         unsigned long a = (unsigned long)addr;
 257
 258         if (unlikely(!validate_addr(ri, a, sizeof(*addr), st))) {
 259                 *err = -QUADD_URC_EACCESS;
 260                 return 0;
 261         }
 262
 263         *err = 0;
 264         return *addr;
 265 }
 266
 267 static inline u16
 268 read_mmap_data_u16(struct ex_region_info *ri,
 269                    const u16 *addr, int st, long *err)
 270 {
 271         unsigned long a = (unsigned long)addr;
 272
 273         if (unlikely(!validate_addr(ri, a, sizeof(*addr), st))) {
 274                 *err = -QUADD_URC_EACCESS;
 275                 return 0;
 276         }
 277
 278         *err = 0;
 279
 280         return get_unaligned(addr);
 281 }
 282
 283 static inline s16
 284 read_mmap_data_s16(struct ex_region_info *ri,
 285                    const s16 *addr, int st, long *err)
 286 {
 287         unsigned long a = (unsigned long)addr;
 288
 289         if (unlikely(!validate_addr(ri, a, sizeof(*addr), st))) {
 290                 *err = -QUADD_URC_EACCESS;
 291                 return 0;
 292         }
 293
 294         *err = 0;
 295
 296         return get_unaligned(addr);
 297 }
 298
 299 static inline u32
 300 read_mmap_data_u32(struct ex_region_info *ri,
 301                    const u32 *addr, int st, long *err)
 302 {
 303         unsigned long a = (unsigned long)addr;
 304
 305         if (unlikely(!validate_addr(ri, a, sizeof(*addr), st))) {
 306                 *err = -QUADD_URC_EACCESS;
 307                 return 0;
 308         }
 309
 310         *err = 0;
 311
 312         return get_unaligned(addr);
 313 }
 314
 315 static inline s32
 316 read_mmap_data_s32(struct ex_region_info *ri,
 317                    const s32 *addr, int st, long *err)
 318 {
 319         unsigned long a = (unsigned long)addr;
 320
 321         if (unlikely(!validate_addr(ri, a, sizeof(*addr), st))) {
 322                 *err = -QUADD_URC_EACCESS;
 323                 return 0;
 324         }
 325
 326         *err = 0;
 327
 328         return get_unaligned(addr);
 329 }
 330
 331 static inline s64
 332 read_mmap_data_s64(struct ex_region_info *ri,
 333                    const s64 *addr, int st, long *err)
 334 {
 335         unsigned long a = (unsigned long)addr;
 336
 337         if (unlikely(!validate_addr(ri, a, sizeof(*addr), st))) {
 338                 *err = -QUADD_URC_EACCESS;
 339                 return 0;
 340         }
 341
 342         *err = 0;
 343
 344         return get_unaligned(addr);
 345 }
 346
 347 static inline u64
 348 read_mmap_data_u64(struct ex_region_info *ri,
 349                    const u64 *addr, int st, long *err)
 350 {
 351         unsigned long a = (unsigned long)addr;
 352
 353         if (unlikely(!validate_addr(ri, a, sizeof(*addr), st))) {
 354                 *err = -QUADD_URC_EACCESS;
 355                 return 0;
 356         }
 357
 358         *err = 0;
 359
 360         return get_unaligned(addr);
 361 }
 362
 363 static inline unsigned long
 364 ex_addr_to_mmap_addr(unsigned long addr,
 365                      struct ex_region_info *ri, int st)
 366 {
 367         unsigned long offset;
 368         struct extab_info *ti;
 369
 370         ti = &ri->ex_sec[st];
 371         offset = addr - ti->addr;
 372
 373         return ti->mmap_offset + offset + (unsigned long)ri->mmap->data;
 374 }
 375
 376 static inline unsigned long
 377 mmap_addr_to_ex_addr(unsigned long addr,
 378                      struct ex_region_info *ri, int st)
 379 {
 380         unsigned long offset;
 381         struct extab_info *ti;
 382
 383         ti = &ri->ex_sec[st];
 384         offset = addr - ti->mmap_offset - (unsigned long)ri->mmap->data;
 385
 386         return ti->addr + offset;
 387 }
 388
 389 static inline int validate_regnum(struct regs_state *rs, int regnum)
 390 {
 391         if (unlikely(regnum >= ARRAY_SIZE(rs->reg))) {
 392                 pr_err_once("error: invalid reg: %d\n", regnum);
 393                 return 0;
 394         }
 395
 396         return 1;
 397 }
 398
 399 static inline void
 400 set_rule_offset(struct regs_state *rs, int regnum, int where, long offset)
 401 {
 402         struct reg_info *r;
 403
 404         if (!validate_regnum(rs, regnum))
 405                 return;
 406
 407         r = &rs->reg[regnum];
 408
 409         r->where = where;
 410         r->loc.offset = offset;
 411 }
 412
 413 static inline void
 414 set_rule_reg(struct regs_state *rs, int regnum, int where, unsigned long reg)
 415 {
 416         struct reg_info *r;
 417
 418         if (!validate_regnum(rs, regnum))
 419                 return;
 420
 421         r = &rs->reg[regnum];
 422
 423         r->where = where;
 424         r->loc.reg = reg;
 425 }
 426
 427 static inline void
 428 set_rule_exp(struct regs_state *rs, int regnum,
 429              int where, const unsigned char *exp)
 430 {
 431         struct reg_info *r;
 432
 433         if (!validate_regnum(rs, regnum))
 434                 return;
 435
 436         r = &rs->reg[regnum];
 437
 438         r->where = where;
 439         r->loc.exp = exp;
 440 }
 441
 442 static inline void
 443 set_rule(struct regs_state *rs, int regnum, int where, long value)
 444 {
 445         set_rule_offset(rs, regnum, where, value);
 446 }
 447
 448 static inline unsigned long
 449 dw_bst_get_initial_loc(const struct dw_fde_table *fi,
 450                        unsigned long data_base)
 451 {
 452         return data_base + fi->initial_loc;
 453 }
 454
 455 static inline unsigned long
 456 dw_bst_get_fde_addr(const struct dw_fde_table *fi,
 457                     unsigned long data_base)
 458 {
 459         return data_base + fi->fde;
 460 }
 461
 462 static inline unsigned long
 463 dwarf_read_uleb128(struct ex_region_info *ri,
 464                    unsigned char *addr,
 465                    unsigned long *ret,
 466                    int st,
 467                    long *err)
 468 {
 469         unsigned long result;
 470         unsigned char byte;
 471         int shift, count;
 472
 473         result = 0;
 474         shift = 0;
 475         count = 0;
 476
 477         while (1) {
 478                 byte = read_mmap_data_u8(ri, addr, st, err);
 479                 if (*err)
 480                         return 0;
 481
 482                 addr++;
 483                 count++;
 484
 485                 result |= (byte & 0x7f) << shift;
 486                 shift += 7;
 487
 488                 if (!(byte & 0x80))
 489                         break;
 490         }
 491
 492         *ret = result;
 493
 494         return count;
 495 }
 496
 497 static inline unsigned long
 498 dwarf_read_sleb128(struct ex_region_info *ri,
 499                    unsigned char *addr,
 500                    long *ret,
 501                    int st,
 502                    long *err)
 503 {
 504         unsigned char byte;
 505         long result, shift;
 506         int num_bits;
 507         int count;
 508
 509         result = 0;
 510         shift = 0;
 511         count = 0;
 512
 513         while (1) {
 514                 byte = read_mmap_data_u8(ri, addr, st, err);
 515                 if (*err)
 516                         return 0;
 517
 518                 addr++;
 519                 result |= (byte & 0x7f) << shift;
 520                 shift += 7;
 521                 count++;
 522
 523                 if (!(byte & 0x80))
 524                         break;
 525         }
 526
 527         num_bits = 8 * sizeof(result);
 528
 529         if ((shift < num_bits) && (byte & 0x40))
 530                 result |= (-1 << shift);
 531
 532         *ret = result;
 533
 534         return count;
 535 }
 536
 537 static inline unsigned int
 538 dw_cfa_opcode(unsigned int insn)
 539 {
 540         return insn & 0xc0;
 541 }
 542
 543 static inline unsigned int
 544 dw_cfa_operand(unsigned int insn)
 545 {
 546         return insn & 0x3f;
 547 }
 548
 549 static int
 550 dwarf_read_encoded_value(struct ex_region_info *ri,
 551                          void *addr,
 552                          void *pcrel_base,
 553                          unsigned long *val,
 554                          char encoding,
 555                          int st)
 556 {
 557         int dw_word_size, count = 0;
 558         long stmp = 0, err = 0;
 559         unsigned long utmp, res = 0;
 560         struct dwarf_cpu_context *cpu_ctx = this_cpu_ptr(ctx.cpu_ctx);
 561
 562         pr_debug("encoding: %#x\n", encoding);
 563
 564         dw_word_size = cpu_ctx->dw_word_size;
 565
 566         if (encoding == DW_EH_PE_omit) {
 567                 pr_debug("DW_EH_PE_omit\n");
 568
 569                 *val = 0;
 570                 return 0;
 571         } else if (encoding == DW_EH_PE_aligned) {
 572                 unsigned long aligned = ALIGN((unsigned long)addr,
 573                                               dw_word_size);
 574
 575                 pr_debug("DW_EH_PE_aligned\n");
 576
 577                 if (dw_word_size == 4) {
 578                         *val = read_mmap_data_u32(ri, (u32 *)aligned, st, &err);
 579                 } else if (dw_word_size == 8) {
 580                         *val = read_mmap_data_u64(ri, (u64 *)aligned, st, &err);
 581                 } else {
 582                         pr_err_once("%s: error: encoding\n", __func__);
 583                         return -QUADD_URC_TBL_IS_CORRUPT;
 584                 }
 585
 586                 if (err)
 587                         return err;
 588
 589                 return dw_word_size;
 590         }
 591
 592         switch (encoding & 0x0f) {
 593         case DW_EH_PE_absptr:
 594                 pr_debug("%s: absptr encoding\n", __func__);
 595
 596                 if (dw_word_size == 4) {
 597                         *val = read_mmap_data_u32(ri, (u32 *)addr, st, &err);
 598                 } else if (dw_word_size == 8) {
 599                         *val = read_mmap_data_u64(ri, (u64 *)addr, st, &err);
 600                 } else {
 601                         pr_err_once("error: wrong dwarf size\n");
 602                         return -QUADD_URC_UNHANDLED_INSTRUCTION;
 603                 }
 604
 605                 if (err)
 606                         return err;
 607
 608                 return dw_word_size;
 609
 610         case DW_EH_PE_sdata2:
 611         case DW_EH_PE_udata2:
 612                 pr_debug("encoding: DW_EH_PE_sdata2\n");
 613                 stmp = read_mmap_data_s16(ri, (s16 *)addr, st, &err);
 614                 if (err)
 615                         return err;
 616
 617                 count += sizeof(s16);
 618                 break;
 619
 620         case DW_EH_PE_sdata4:
 621         case DW_EH_PE_udata4:
 622                 pr_debug("encoding: DW_EH_PE_udata4/sdata4\n");
 623                 stmp = read_mmap_data_s32(ri, (s32 *)addr, st, &err);
 624                 if (err)
 625                         return err;
 626
 627                 count += sizeof(s32);
 628                 break;
 629
 630         case DW_EH_PE_sdata8:
 631         case DW_EH_PE_udata8:
 632                 pr_debug("encoding: DW_EH_PE_udata8\n");
 633                 stmp = read_mmap_data_s64(ri, (s64 *)addr, st, &err);
 634                 if (err)
 635                         return err;
 636
 637                 count += sizeof(s64);
 638                 break;
 639
 640         case DW_EH_PE_uleb128:
 641                 pr_debug("encoding: DW_EH_PE_uleb128\n");
 642                 count += dwarf_read_uleb128(ri, addr, &utmp, st, &err);
 643                 if (err)
 644                         return err;
 645
 646                 stmp = utmp;
 647                 break;
 648
 649         case DW_EH_PE_sleb128:
 650                 pr_debug("encoding: DW_EH_PE_sleb128\n");
 651                 count += dwarf_read_sleb128(ri, addr, &stmp, st, &err);
 652                 if (err)
 653                         return err;
 654
 655                 break;
 656
 657         default:
 658                 pr_warn_once("%s: warning: encoding: %#x\n",
 659                              __func__, encoding & 0x0f);
 660                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
 661         }
 662
 663         switch (encoding & 0x70) {
 664         case DW_EH_PE_absptr:
 665                 pr_debug("DW_EH_PE_absptr\n");
 666                 res = stmp;
 667                 break;
 668
 669         case DW_EH_PE_pcrel:
 670                 pr_debug("DW_EH_PE_pcrel, pcrel_base: %p, stmp: %ld\n",
 671                          pcrel_base, stmp);
 672                 res = (unsigned long)pcrel_base + stmp;
 673                 break;
 674
 675         case DW_EH_PE_textrel:
 676                 pr_warn_once("warning: DW_EH_PE_textrel\n");
 677                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
 678
 679         case DW_EH_PE_datarel:
 680                 pr_warn_once("warning: DW_EH_PE_datarel\n");
 681                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
 682
 683         case DW_EH_PE_funcrel:
 684                 pr_warn_once("warning: DW_EH_PE_funcrel\n");
 685                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
 686
 687         default:
 688                 pr_warn_once("%s: warning: encoding: %#x\n",
 689                              __func__, encoding & 0x70);
 690                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
 691         }
 692
 693         if (res != 0) {
 694                 if (encoding & DW_EH_PE_indirect) {
 695                         pr_debug("DW_EH_PE_indirect\n");
 696
 697                         if (dw_word_size == 4) {
 698                                 res = read_mmap_data_u32(ri, (u32 *)res,
 699                                                          st, &err);
 700                         } else if (dw_word_size == 8) {
 701                                 res = read_mmap_data_u64(ri, (u64 *)res,
 702                                                          st, &err);
 703                         } else {
 704                                 pr_err_once("error: wrong dwarf size\n");
 705                                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
 706                         }
 707
 708                         /* we ignore links to unloaded sections */
 709                         if (err)
 710                                 res = 0;
 711                 }
 712         }
 713
 714         *val = res;
 715
 716         return count;
 717 }
 718
 719 static long
 720 dwarf_cfa_exec_insns(struct ex_region_info *ri,
 721                      unsigned char *insn_start,
 722                      unsigned char *insn_end,
 723                      struct dw_cie *cie,
 724                      struct stackframe *sf,
 725                      unsigned long pc)
 726 {
 727         unsigned char insn;
 728         unsigned char *c_insn;
 729         unsigned int expr_len, delta;
 730         unsigned long utmp, reg;
 731         long offset, stmp, err = 0;
 732         struct regs_state *rs, *rs_initial, *rs_stack;
 733         struct dwarf_cpu_context *cpu_ctx = this_cpu_ptr(ctx.cpu_ctx);
 734
 735         rs = &sf->rs;
 736         rs_initial = &sf->rs_initial;
 737
 738         rs_stack = cpu_ctx->rs_stack;
 739         cpu_ctx->depth = 0;
 740
 741         c_insn = insn_start;
 742
 743         while (c_insn < insn_end && sf->pc <= pc) {
 744                 insn = read_mmap_data_u8(ri, c_insn++,
 745                                          QUADD_SEC_TYPE_EH_FRAME, &err);
 746                 if (err)
 747                         return err;
 748
 749                 switch (dw_cfa_opcode(insn)) {
 750                 case DW_CFA_advance_loc:
 751                         delta = dw_cfa_operand(insn);
 752                         delta *= cie->code_align_factor;
 753                         sf->pc += delta;
 754                         pr_debug("DW_CFA_advance_loc: pc: %#lx --> %#lx (delta: %#x)\n",
 755                                 sf->pc - delta, sf->pc, delta);
 756                         continue;
 757
 758                 case DW_CFA_offset:
 759                         reg = dw_cfa_operand(insn);
 760                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 761                                                      QUADD_SEC_TYPE_EH_FRAME,
 762                                                      &err);
 763                         if (err)
 764                                 return err;
 765
 766                         offset = utmp * cie->data_align_factor;
 767                         set_rule_offset(rs, reg, DW_WHERE_CFAREL, offset);
 768                         pr_debug("DW_CFA_offset: reg: r%lu, offset(addr): %#lx (%ld)\n",
 769                                 reg, offset, offset);
 770                         continue;
 771
 772                 case DW_CFA_restore:
 773                         reg = dw_cfa_operand(insn);
 774
 775                         if (!validate_regnum(rs, reg))
 776                                 break;
 777
 778                         rs->reg[reg] = rs_initial->reg[reg];
 779                         pr_debug("DW_CFA_restore: reg: r%lu\n", reg);
 780                         continue;
 781                 }
 782
 783                 switch (insn) {
 784                 case DW_CFA_nop:
 785                         pr_debug("DW_CFA_nop\n");
 786                         continue;
 787
 788                 case DW_CFA_advance_loc1:
 789                         delta = read_mmap_data_u8(ri, c_insn++,
 790                                                   QUADD_SEC_TYPE_EH_FRAME,
 791                                                   &err);
 792                         if (err)
 793                                 return err;
 794
 795                         sf->pc += delta * cie->code_align_factor;
 796                         pr_debug("DW_CFA_advance_loc1: pc: %#lx --> %#lx (delta: %#lx)\n",
 797                                 sf->pc - delta * cie->code_align_factor, sf->pc,
 798                                 delta * cie->code_align_factor);
 799                         break;
 800
 801                 case DW_CFA_advance_loc2:
 802                         delta = read_mmap_data_u16(ri, (u16 *)c_insn,
 803                                                    QUADD_SEC_TYPE_EH_FRAME,
 804                                                    &err);
 805                         if (err)
 806                                 return err;
 807
 808                         c_insn += 2;
 809                         sf->pc += delta * cie->code_align_factor;
 810                         pr_debug("DW_CFA_advance_loc2: pc: %#lx --> %#lx (delta: %#lx)\n",
 811                                 sf->pc - delta * cie->code_align_factor, sf->pc,
 812                                 delta * cie->code_align_factor);
 813                         break;
 814
 815                 case DW_CFA_advance_loc4:
 816                         delta = read_mmap_data_u32(ri, (u32 *)c_insn,
 817                                                    QUADD_SEC_TYPE_EH_FRAME,
 818                                                    &err);
 819                         if (err)
 820                                 return err;
 821
 822                         c_insn += 4;
 823                         sf->pc += delta * cie->code_align_factor;
 824                         pr_debug("DW_CFA_advance_loc4: pc: %#lx --> %#lx (delta: %#lx)\n",
 825                                 sf->pc - delta * cie->code_align_factor, sf->pc,
 826                                 delta * cie->code_align_factor);
 827                         break;
 828
 829                 case DW_CFA_offset_extended:
 830                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 831                                                      QUADD_SEC_TYPE_EH_FRAME,
 832                                                      &err);
 833                         if (err)
 834                                 return err;
 835
 836                         reg = utmp;
 837                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 838                                                      QUADD_SEC_TYPE_EH_FRAME,
 839                                                      &err);
 840                         if (err)
 841                                 return err;
 842
 843                         offset = utmp * cie->data_align_factor;
 844                         pr_debug("DW_CFA_offset_extended: reg: r%lu, offset: %#lx\n",
 845                                  reg, offset);
 846                         break;
 847
 848                 case DW_CFA_restore_extended:
 849                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
 850                                                      QUADD_SEC_TYPE_EH_FRAME,
 851                                                      &err);
 852                         if (err)
 853                                 return err;
 854
 855                         pr_debug("DW_CFA_restore_extended: reg: r%lu\n", reg);
 856                         break;
 857
 858                 case DW_CFA_undefined:
 859                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
 860                                                      QUADD_SEC_TYPE_EH_FRAME,
 861                                                      &err);
 862                         if (err)
 863                                 return err;
 864
 865                         set_rule(rs, reg, DW_WHERE_UNDEF, 0);
 866                         pr_debug("DW_CFA_undefined: reg: r%lu\n", reg);
 867                         break;
 868
 869                 case DW_CFA_def_cfa:
 870                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 871                                                      QUADD_SEC_TYPE_EH_FRAME,
 872                                                      &err);
 873                         if (err)
 874                                 return err;
 875
 876                         rs->cfa_register = utmp;
 877                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 878                                                      QUADD_SEC_TYPE_EH_FRAME,
 879                                                      &err);
 880                         if (err)
 881                                 return err;
 882
 883                         rs->cfa_offset = utmp;
 884                         pr_debug("DW_CFA_def_cfa: cfa_register: r%u, cfa_offset: %ld (%#lx)\n",
 885                                  rs->cfa_register, rs->cfa_offset,
 886                                  rs->cfa_offset);
 887                         break;
 888
 889                 case DW_CFA_def_cfa_register:
 890                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 891                                                      QUADD_SEC_TYPE_EH_FRAME,
 892                                                      &err);
 893                         if (err)
 894                                 return err;
 895
 896                         rs->cfa_register = utmp;
 897                         pr_debug("DW_CFA_def_cfa_register: cfa_register: r%u\n",
 898                                rs->cfa_register);
 899                         break;
 900
 901                 case DW_CFA_def_cfa_offset:
 902                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 903                                                      QUADD_SEC_TYPE_EH_FRAME,
 904                                                      &err);
 905                         if (err)
 906                                 return err;
 907
 908                         rs->cfa_offset = utmp;
 909                         pr_debug("DW_CFA_def_cfa_offset: cfa_offset: %ld (%#lx)\n",
 910                                rs->cfa_offset, rs->cfa_offset);
 911                         break;
 912
 913                 case DW_CFA_def_cfa_expression:
 914                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 915                                                      QUADD_SEC_TYPE_EH_FRAME,
 916                                                      &err);
 917                         if (err)
 918                                 return err;
 919
 920                         expr_len = utmp;
 921
 922                         rs->cfa_expr = c_insn;
 923                         rs->cfa_expr_len = expr_len;
 924                         rs->cfa_how = DW_CFA_EXP;
 925                         c_insn += expr_len;
 926
 927                         pr_debug("DW_CFA_def_cfa_expression: expr_len: %#x\n",
 928                                  expr_len);
 929                         break;
 930
 931                 case DW_CFA_expression:
 932                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
 933                                                      QUADD_SEC_TYPE_EH_FRAME,
 934                                                      &err);
 935                         if (err)
 936                                 return err;
 937
 938                         set_rule_exp(rs, reg, DW_WHERE_EXPR, c_insn);
 939
 940                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 941                                                      QUADD_SEC_TYPE_EH_FRAME,
 942                                                      &err);
 943                         if (err)
 944                                 return err;
 945
 946                         c_insn += utmp;
 947
 948                         pr_debug("DW_CFA_expression: reg: r%lu\n", reg);
 949                         break;
 950
 951                 case DW_CFA_offset_extended_sf:
 952                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
 953                                                      QUADD_SEC_TYPE_EH_FRAME,
 954                                                      &err);
 955                         if (err)
 956                                 return err;
 957
 958                         c_insn += dwarf_read_sleb128(ri, c_insn, &stmp,
 959                                                      QUADD_SEC_TYPE_EH_FRAME,
 960                                                      &err);
 961                         if (err)
 962                                 return err;
 963
 964                         offset = stmp * cie->data_align_factor;
 965                         set_rule_offset(rs, reg, DW_WHERE_CFAREL, offset);
 966                         pr_debug("DW_CFA_offset_extended_sf: reg: r%lu, offset: %#lx\n",
 967                                  reg, offset);
 968                         break;
 969
 970                 case DW_CFA_val_offset:
 971                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
 972                                                      QUADD_SEC_TYPE_EH_FRAME,
 973                                                      &err);
 974                         if (err)
 975                                 return err;
 976
 977                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
 978                                                      QUADD_SEC_TYPE_EH_FRAME,
 979                                                      &err);
 980                         if (err)
 981                                 return err;
 982
 983                         offset = utmp * cie->data_align_factor;
 984                         set_rule_offset(rs, reg, DW_WHERE_VAL_OFFSET, offset);
 985                         pr_debug("DW_CFA_val_offset: reg: r%lu, offset(addr): %#lx\n",
 986                                  reg, offset);
 987                         break;
 988
 989                 case DW_CFA_val_offset_sf:
 990                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
 991                                 QUADD_SEC_TYPE_EH_FRAME, &err);
 992                         if (err)
 993                                 return err;
 994
 995                         c_insn += dwarf_read_sleb128(ri, c_insn, &stmp,
 996                                                      QUADD_SEC_TYPE_EH_FRAME,
 997                                                      &err);
 998                         if (err)
 999                                 return err;
1000
1001                         offset = stmp * cie->data_align_factor;
1002                         set_rule_offset(rs, reg, DW_WHERE_VAL_OFFSET, offset);
1003                         pr_debug("DW_CFA_val_offset_sf: reg: r%lu, offset(addr): %#lx\n",
1004                                  reg, offset);
1005                         break;
1006
1007                 case DW_CFA_GNU_args_size:
1008                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
1009                                                      QUADD_SEC_TYPE_EH_FRAME,
1010                                                      &err);
1011                         if (err)
1012                                 return err;
1013
1014                         pr_debug("DW_CFA_GNU_args_size: offset: %#lx\n", utmp);
1015                         break;
1016
1017                 case DW_CFA_GNU_negative_offset_extended:
1018                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
1019                                                      QUADD_SEC_TYPE_EH_FRAME,
1020                                                      &err);
1021                         if (err)
1022                                 return err;
1023
1024                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
1025                                                      QUADD_SEC_TYPE_EH_FRAME,
1026                                                      &err);
1027                         if (err)
1028                                 return err;
1029
1030                         offset = utmp * cie->data_align_factor;
1031                         set_rule_offset(rs, reg, DW_WHERE_CFAREL, -offset);
1032                         pr_debug("DW_CFA_GNU_negative_offset_extended: reg: r%lu, offset: %#lx\n",
1033                                  reg, offset);
1034                         break;
1035
1036                 case DW_CFA_remember_state:
1037                         pr_debug("DW_CFA_remember_state\n");
1038
1039                         if (cpu_ctx->depth >= DW_MAX_RS_STACK_DEPTH) {
1040                                 pr_warn_once("error: rs stack was overflowed\n");
1041                                 return 0;
1042                         }
1043
1044                         rs_stack[cpu_ctx->depth++] = *rs;
1045                         break;
1046
1047                 case DW_CFA_restore_state:
1048                         pr_debug("DW_CFA_restore_state\n");
1049
1050                         if (cpu_ctx->depth == 0) {
1051                                 pr_warn_once("error: rs stack error\n");
1052                                 return 0;
1053                         }
1054
1055                         *rs = rs_stack[--cpu_ctx->depth];
1056                         break;
1057
1058                 case DW_CFA_def_cfa_sf:
1059                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
1060                                                      QUADD_SEC_TYPE_EH_FRAME,
1061                                                      &err);
1062                         if (err)
1063                                 return err;
1064
1065                         c_insn += dwarf_read_sleb128(ri, c_insn, &stmp,
1066                                                      QUADD_SEC_TYPE_EH_FRAME,
1067                                                      &err);
1068                         if (err)
1069                                 return err;
1070
1071                         rs->cfa_register = utmp;
1072                         rs->cfa_offset = stmp * cie->data_align_factor;
1073                         rs->cfa_how = DW_CFA_REG_OFFSET;
1074
1075                         pr_debug("DW_CFA_def_cfa_sf: cfa_register: r%u, cfa_offset: %ld (%#lx)\n",
1076                                 rs->cfa_register, rs->cfa_offset,
1077                                 rs->cfa_offset);
1078                         break;
1079
1080                 case DW_CFA_def_cfa_offset_sf:
1081                         c_insn += dwarf_read_sleb128(ri, c_insn, &stmp,
1082                                                      QUADD_SEC_TYPE_EH_FRAME,
1083                                                      &err);
1084                         if (err)
1085                                 return err;
1086
1087                         rs->cfa_offset = stmp * cie->data_align_factor;
1088                         pr_debug("DW_CFA_def_cfa_offset_sf: cfa_offset: %ld (%#lx)\n",
1089                                 rs->cfa_offset, rs->cfa_offset);
1090                         break;
1091
1092                 case DW_CFA_same_value:
1093                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
1094                                                      QUADD_SEC_TYPE_EH_FRAME,
1095                                                      &err);
1096                         if (err)
1097                                 return err;
1098
1099                         set_rule(rs, reg, DW_WHERE_SAME, 0);
1100                         pr_debug("DW_CFA_same_value: reg: r%lu\n", reg);
1101                         break;
1102
1103                 case DW_CFA_val_expression:
1104                         c_insn += dwarf_read_uleb128(ri, c_insn, &reg,
1105                                                      QUADD_SEC_TYPE_EH_FRAME,
1106                                                      &err);
1107                         if (err)
1108                                 return err;
1109
1110                         set_rule_exp(rs, reg, DW_WHERE_VAL_EXPR, c_insn);
1111                         c_insn += dwarf_read_uleb128(ri, c_insn, &utmp,
1112                                                      QUADD_SEC_TYPE_EH_FRAME,
1113                                                      &err);
1114                         if (err)
1115                                 return err;
1116
1117                         c_insn += utmp;
1118                         pr_debug("DW_CFA_val_expression: reg: r%lu\n", reg);
1119                         break;
1120
1121                 default:
1122                         pr_warn_once("warning: unhandled dwarf instr %#x\n",
1123                                      insn);
1124                         break;
1125                 }
1126         }
1127
1128         return 0;
1129 }
1130
1131 static long
1132 decode_cie_entry(struct ex_region_info *ri,
1133                  struct dw_cie *cie,
1134                  unsigned char *entry,
1135                  size_t length)
1136 {
1137         long err;
1138         unsigned long utmp;
1139         unsigned char *p, *end, *aug;
1140         unsigned int cie_version, id, len, max_len;
1141
1142         p = entry;
1143         end = entry + length;
1144
1145         p += sizeof(u32);
1146
1147         id = read_mmap_data_u32(ri, (u32 *)p, QUADD_SEC_TYPE_EH_FRAME, &err);
1148         if (err)
1149                 return err;
1150
1151         p += sizeof(u32);
1152
1153         if (id != 0)
1154                 return -QUADD_URC_TBL_IS_CORRUPT;
1155
1156         cie_version = read_mmap_data_u8(ri, p++, QUADD_SEC_TYPE_EH_FRAME, &err);
1157         if (err)
1158                 return err;
1159
1160         if (cie_version != 1 && cie_version != 3) {
1161                 pr_err_once("error: wrong cie_version: %u\n", cie_version);
1162                 return -QUADD_URC_TBL_IS_CORRUPT;
1163         }
1164
1165         if (p >= end)
1166                 return -QUADD_URC_TBL_IS_CORRUPT;
1167
1168         max_len = end - p - 1;
1169         len = strnlen((const char *)p, max_len);
1170         if (len == max_len)
1171                 return -QUADD_URC_TBL_IS_CORRUPT;
1172
1173         cie->aug_string = p;
1174         p += len + 1;
1175
1176         pr_debug("aug_string: %s\n", cie->aug_string);
1177
1178         p += dwarf_read_uleb128(ri, p, &cie->code_align_factor,
1179                                 QUADD_SEC_TYPE_EH_FRAME, &err);
1180         if (err)
1181                 return err;
1182
1183         p += dwarf_read_sleb128(ri, p, &cie->data_align_factor,
1184                                 QUADD_SEC_TYPE_EH_FRAME, &err);
1185         if (err)
1186                 return err;
1187
1188         if (cie_version == 1) {
1189                 cie->retaddr_reg = read_mmap_data_u8(ri, p++,
1190                                                      QUADD_SEC_TYPE_EH_FRAME,
1191                                                      &err);
1192                 if (err)
1193                         return err;
1194         } else {
1195                 p += dwarf_read_uleb128(ri, p, &utmp,
1196                                         QUADD_SEC_TYPE_EH_FRAME, &err);
1197                 if (err)
1198                         return err;
1199
1200                 cie->retaddr_reg = utmp;
1201         }
1202
1203         pr_debug("address column: %u\n", cie->retaddr_reg);
1204
1205         aug = cie->aug_string;
1206         cie->z_aug = 0;
1207
1208         cie->initial_insn = NULL;
1209         cie->initial_insn_len = 0;
1210
1211         if (*aug == 'z') {
1212                 p += dwarf_read_uleb128(ri, p, &cie->aug_size,
1213                                         QUADD_SEC_TYPE_EH_FRAME, &err);
1214                 if (err)
1215                         return err;
1216
1217                 cie->initial_insn = p + cie->aug_size;
1218                 aug++;
1219
1220                 cie->z_aug = 1;
1221         } else {
1222                 pr_warn_once("warning: !aug_z\n");
1223         }
1224
1225         cie->fde_encoding = 0;
1226         cie->lsda_encoding = DW_EH_PE_omit;
1227         cie->personality = NULL;
1228
1229         while (*aug != '\0') {
1230                 if (p >= end)
1231                         return -QUADD_URC_TBL_IS_CORRUPT;
1232
1233                 if (*aug == 'L') {
1234                         cie->lsda_encoding =
1235                                 read_mmap_data_u8(ri, p++,
1236                                                   QUADD_SEC_TYPE_EH_FRAME,
1237                                                   &err);
1238                         if (err)
1239                                 return err;
1240
1241                         aug++;
1242                 } else if (*aug == 'R') {
1243                         cie->fde_encoding =
1244                                 read_mmap_data_u8(ri, p++,
1245                                                   QUADD_SEC_TYPE_EH_FRAME,
1246                                                   &err);
1247                         if (err)
1248                                 return err;
1249
1250                         aug++;
1251                         pr_debug("fde_encoding: %#x\n", cie->fde_encoding);
1252                 } else if (*aug == 'P') {
1253                         int cnt;
1254                         void *pcrel_base;
1255                         unsigned char handler_encoding;
1256                         unsigned long personality;
1257
1258                         handler_encoding = *p++;
1259
1260                         pcrel_base = (void *)
1261                                 mmap_addr_to_ex_addr((unsigned long)p,
1262                                                      ri,
1263                                                      QUADD_SEC_TYPE_EH_FRAME);
1264
1265                         cnt = dwarf_read_encoded_value(ri, p, pcrel_base,
1266                                                        &personality,
1267                                                        handler_encoding,
1268                                                        QUADD_SEC_TYPE_EH_FRAME);
1269                         if (cnt < 0) {
1270                                 pr_err_once("%s: error: personality routine\n",
1271                                             __func__);
1272                                 return cnt;
1273                         }
1274                         p += cnt;
1275
1276                         pr_debug("personality: %#lx\n", personality);
1277                         cie->personality = (void *)personality;
1278                         aug++;
1279                 } else if (*aug == 'S') {
1280                         aug++;
1281                         pr_debug("%s: aug: S\n", __func__);
1282                 } else {
1283                         pr_warn_once("%s: warning: unknown aug\n", __func__);
1284                         return -QUADD_URC_UNHANDLED_INSTRUCTION;
1285                 }
1286         }
1287
1288         if (p > end) {
1289                 pr_err_once("%s: error: cie\n", __func__);
1290                 return -QUADD_URC_TBL_IS_CORRUPT;
1291         }
1292
1293         if (p == end)
1294                 return 0;
1295
1296         if (!cie->initial_insn)
1297                 cie->initial_insn = p;
1298
1299         cie->initial_insn_len = end - cie->initial_insn;
1300
1301         return 0;
1302 }
1303
1304 static long
1305 decode_fde_entry(struct ex_region_info *ri,
1306                  struct dw_fde *fde,
1307                  unsigned char *entry,
1308                  size_t length)
1309 {
1310         int count;
1311         long err = 0;
1312         unsigned long utmp;
1313         unsigned char *p, *end, *pcrel_base;
1314         struct dw_cie *cie = fde->cie;
1315
1316         p = entry;
1317         end = entry + length;
1318
1319         p += sizeof(u32);
1320         p += sizeof(u32);
1321
1322         pcrel_base = (unsigned char *)
1323                 mmap_addr_to_ex_addr((unsigned long)p, ri,
1324                                      QUADD_SEC_TYPE_EH_FRAME);
1325
1326         count = dwarf_read_encoded_value(ri, p, pcrel_base,
1327                                          &fde->initial_location,
1328                                          cie->fde_encoding,
1329                                          QUADD_SEC_TYPE_EH_FRAME);
1330         if (count < 0)
1331                 return count;
1332
1333         p += count;
1334
1335         fde->address_range = read_mmap_data_u32(ri, (u32 *)p,
1336                                                 QUADD_SEC_TYPE_EH_FRAME, &err);
1337         if (err)
1338                 return err;
1339
1340         p += sizeof(u32);
1341
1342         pr_debug("init location: %#lx\n", fde->initial_location);
1343         pr_debug("address_range: %#lx\n", fde->address_range);
1344
1345         if (cie->z_aug) {
1346                 p += dwarf_read_uleb128(ri, p, &utmp,
1347                                         QUADD_SEC_TYPE_EH_FRAME, &err);
1348                 if (err)
1349                         return err;
1350
1351                 p += utmp;
1352         }
1353
1354         if (p > end) {
1355                 pr_err_once("%s: error: incorrect fde\n", __func__);
1356                 return -QUADD_URC_TBL_IS_CORRUPT;
1357         }
1358
1359         fde->insn_length = end - p;
1360
1361         if (fde->insn_length > 0)
1362                 fde->instructions = p;
1363         else
1364                 fde->instructions = NULL;
1365
1366         return 0;
1367 }
1368
1369 static const struct dw_fde_table *
1370 dwarf_bst_find_idx(unsigned long data_base,
1371                    struct dw_fde_table *fde_table,
1372                    unsigned long length,
1373                    unsigned long addr)
1374 {
1375         unsigned long initial_loc;
1376         struct dw_fde_table *start, *stop;
1377         struct dw_fde_table *mid = NULL;
1378
1379         if (unlikely(!length))
1380                 return NULL;
1381
1382         start = fde_table;
1383         stop = start + length - 1;
1384
1385         initial_loc = dw_bst_get_initial_loc(start, data_base);
1386         if (addr < initial_loc)
1387                 return NULL;
1388
1389         initial_loc = dw_bst_get_initial_loc(stop, data_base);
1390         if (addr >= initial_loc)
1391                 return NULL;
1392
1393         while (start < stop - 1) {
1394                 mid = start + ((stop - start) >> 1);
1395
1396                 initial_loc = dw_bst_get_initial_loc(mid, data_base);
1397
1398                 if (addr < initial_loc)
1399                         stop = mid;
1400                 else
1401                         start = mid;
1402         }
1403
1404         return start;
1405 }
1406
1407 static struct dw_fde_table *
1408 dwarf_get_bs_table(struct ex_region_info *ri,
1409                    void *data,
1410                    unsigned long length,
1411                    unsigned long data_base,
1412                    unsigned long *nr_entries)
1413 {
1414         int count;
1415         unsigned char *p, *end;
1416         struct dw_fde_table *bst;
1417         unsigned long fde_count, frame_ptr;
1418         struct dw_eh_frame_hdr *hdr = data;
1419
1420         if (length <= sizeof(*hdr))
1421                 return NULL;
1422
1423         end = data + length;
1424
1425         pr_debug("hdr: %p\n", hdr);
1426
1427         if (hdr->version != 1) {
1428                 pr_warn_once("warning: unknown eh hdr format\n");
1429                 return NULL;
1430         }
1431         p = (unsigned char *)(hdr + 1);
1432
1433         if (hdr->eh_frame_ptr_enc != DW_EH_PE_omit) {
1434                 count = dwarf_read_encoded_value(ri, p, (void *)data_base,
1435                                                  &frame_ptr,
1436                                                  hdr->eh_frame_ptr_enc,
1437                                                  QUADD_SEC_TYPE_EH_FRAME_HDR);
1438                 if (count < 0)
1439                         return NULL;
1440
1441                 p += count;
1442         }
1443
1444         if (hdr->fde_count_enc == DW_EH_PE_omit)
1445                 return NULL;
1446
1447         count = dwarf_read_encoded_value(ri, p, (void *)data_base,
1448                                          &fde_count, hdr->fde_count_enc,
1449                                          QUADD_SEC_TYPE_EH_FRAME_HDR);
1450         if (count < 0)
1451                 return NULL;
1452
1453         p += count;
1454
1455         if (p >= end)
1456                 return NULL;
1457
1458         if (fde_count * sizeof(*bst) !=  end - p)
1459                 return NULL;
1460
1461         if (hdr->table_enc != (DW_EH_PE_datarel | DW_EH_PE_sdata4)) {
1462                 pr_warn_once("warning: unknown eh hdr format\n");
1463                 return NULL;
1464         }
1465
1466         bst = (struct dw_fde_table *)p;
1467         *nr_entries = fde_count;
1468
1469         return bst;
1470 }
1471
1472 static long
1473 dwarf_decode_fde_cie(struct ex_region_info *ri,
1474                      unsigned char *fde_p,
1475                      struct dw_cie *cie,
1476                      struct dw_fde *fde)
1477 {
1478         u32 *p;
1479         long err;
1480         unsigned char *cie_p;
1481         unsigned long cie_pointer, length;
1482         unsigned char *frame_start;
1483         unsigned long frame_len, addr;
1484         struct extab_info *ti;
1485
1486         ti = &ri->ex_sec[QUADD_SEC_TYPE_EH_FRAME];
1487
1488         addr = ti->addr;
1489
1490         frame_start = (unsigned char *)
1491                 ex_addr_to_mmap_addr(addr, ri, QUADD_SEC_TYPE_EH_FRAME);
1492
1493         frame_len = ti->length;
1494
1495         pr_debug("eh frame: %p - %p\n",
1496                  frame_start, frame_start + frame_len);
1497
1498         p = (u32 *)fde_p;
1499
1500         length = read_mmap_data_u32(ri, p++, QUADD_SEC_TYPE_EH_FRAME, &err);
1501         if (err)
1502                 return err;
1503
1504         if (length == 0xffffffff) {
1505                 pr_warn_once("warning: 64-bit .eh_frame is not supported\n");
1506                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
1507         }
1508
1509         fde->offset = fde_p - frame_start;
1510         fde->length = length + sizeof(u32);
1511
1512         pr_debug("FDE: fde_p: %p, offset: %#lx, len: %#lx\n",
1513                  fde_p, fde->offset, fde->length);
1514
1515         cie_pointer = read_mmap_data_u32(ri, p, QUADD_SEC_TYPE_EH_FRAME, &err);
1516         if (err)
1517                 return err;
1518
1519         fde->cie_pointer = cie_pointer;
1520         cie_p = (unsigned char *)p - cie_pointer;
1521
1522         length = read_mmap_data_u32(ri, (u32 *)cie_p,
1523                                     QUADD_SEC_TYPE_EH_FRAME, &err);
1524         if (err)
1525                 return err;
1526
1527         if (length == 0xffffffff) {
1528                 pr_warn_once("warning: 64-bit .eh_frame is not supported\n");
1529                 return -QUADD_URC_UNHANDLED_INSTRUCTION;
1530         }
1531
1532         cie->offset = cie_p - frame_start;
1533         cie->length = length + sizeof(u32);
1534
1535         pr_debug("CIE: cie_p: %p, offset: %#lx, len: %#lx\n",
1536                  cie_p, cie->offset, cie->length);
1537
1538         err = decode_cie_entry(ri, cie, cie_p, cie->length);
1539         if (err < 0)
1540                 return err;
1541
1542         fde->cie = cie;
1543
1544         err = decode_fde_entry(ri, fde, fde_p, fde->length);
1545         if (err < 0)
1546                 return err;
1547
1548         return 0;
1549 }
1550
1551 static void *
1552 dwarf_find_fde(struct ex_region_info *ri,
1553                void *data,
1554                unsigned long length,
1555                unsigned long pc)
1556 {
1557         long err;
1558         const struct dw_fde_table *fi;
1559         unsigned long fde_count = 0, data_base;
1560         unsigned long fde_addr, init_loc;
1561         struct dw_fde_table *bst;
1562         struct extab_info *ti;
1563
1564         ti = &ri->ex_sec[QUADD_SEC_TYPE_EH_FRAME_HDR];
1565         data_base = ti->addr;
1566
1567         bst = dwarf_get_bs_table(ri, data, length, data_base, &fde_count);
1568         if (!bst || fde_count == 0) {
1569                 pr_warn_once("warning: bs_table\n");
1570                 return NULL;
1571         }
1572
1573         fi = &bst[fde_count - 1];
1574         init_loc = dw_bst_get_initial_loc(fi, data_base);
1575
1576         if (pc >= init_loc) {
1577                 unsigned long start, end;
1578
1579                 fde_addr = dw_bst_get_fde_addr(fi, data_base);
1580                 fde_addr = ex_addr_to_mmap_addr(fde_addr, ri,
1581                                                 QUADD_SEC_TYPE_EH_FRAME);
1582
1583                 if (pc == init_loc)
1584                         return (void *)fde_addr;
1585
1586                 if (ri->tf_end > 0) {
1587                         start = ri->tf_start;
1588                         end = ri->tf_end;
1589                 } else {
1590                         struct dw_cie cie;
1591                         struct dw_fde fde;
1592
1593                         err = dwarf_decode_fde_cie(ri, (void *)fde_addr,
1594                                                    &cie, &fde);
1595                         if (err < 0)
1596                                 return NULL;
1597
1598                         start = fde.initial_location;
1599                         end = start + fde.address_range;
1600
1601                         quadd_unwind_set_tail_info(ri->vm_start, start, end);
1602                 }
1603
1604                 return (pc >= start && pc < end) ?
1605                        (void *)fde_addr : NULL;
1606         }
1607
1608         fi = dwarf_bst_find_idx(data_base, bst, fde_count, pc);
1609         if (!fi)
1610                 return NULL;
1611
1612         fde_addr = dw_bst_get_fde_addr(fi, data_base);
1613         fde_addr = ex_addr_to_mmap_addr(fde_addr, ri, QUADD_SEC_TYPE_EH_FRAME);
1614
1615         return (void *)fde_addr;
1616 }
1617
1618 static long
1619 dwarf_decode(struct ex_region_info *ri,
1620              struct dw_cie *cie,
1621              struct dw_fde *fde,
1622              unsigned long pc)
1623 {
1624         long err;
1625         unsigned char *fde_p;
1626         unsigned char *hdr_start;
1627         unsigned long hdr_len, addr;
1628         struct extab_info *ti;
1629
1630         ti = &ri->ex_sec[QUADD_SEC_TYPE_EH_FRAME_HDR];
1631
1632         addr = ti->addr;
1633
1634         hdr_start = (unsigned char *)
1635                 ex_addr_to_mmap_addr(addr, ri, QUADD_SEC_TYPE_EH_FRAME_HDR);
1636
1637         hdr_len = ti->length;
1638
1639         pr_debug("eh frame hdr: %p - %p\n",
1640                  hdr_start, hdr_start + hdr_len);
1641
1642         fde_p = dwarf_find_fde(ri, hdr_start, hdr_len, pc);
1643         if (!fde_p)
1644                 return -QUADD_URC_IDX_NOT_FOUND;
1645
1646         err = dwarf_decode_fde_cie(ri, fde_p, cie, fde);
1647         if (err < 0)
1648                 return err;
1649
1650         if (pc < fde->initial_location ||
1651             pc >= fde->initial_location + fde->address_range) {
1652                 pr_debug("pc is not in range: %#lx - %#lx\n",
1653                          fde->initial_location,
1654                          fde->initial_location + fde->address_range);
1655                 return -QUADD_URC_IDX_NOT_FOUND;
1656         }
1657
1658         return 0;
1659 }
1660
1661 static long def_cfa(struct stackframe *sf, struct regs_state *rs)
1662 {
1663         int reg = rs->cfa_register;
1664
1665         if (reg >= 0) {
1666                 if (reg >= QUADD_NUM_REGS)
1667                         return -QUADD_URC_TBL_IS_CORRUPT;
1668
1669                 pr_debug("r%d --> cfa (%#lx)\n", reg, sf->cfa);
1670                 sf->cfa = sf->vregs[reg];
1671         }
1672
1673         sf->cfa += rs->cfa_offset;
1674         pr_debug("cfa += %#lx (%#lx)\n", rs->cfa_offset, sf->cfa);
1675
1676         return 0;
1677 }
1678
1679 static long
1680 unwind_frame(struct ex_region_info *ri,
1681              struct stackframe *sf,
1682              struct vm_area_struct *vma_sp,
1683              unsigned int *unw_type)
1684 {
1685         int i;
1686         long err;
1687         unsigned char *insn_end;
1688         unsigned long addr, return_addr, val, user_reg_size;
1689         struct dw_fde fde;
1690         struct dw_cie cie;
1691         unsigned long pc = sf->pc;
1692         struct regs_state *rs, *rs_initial;
1693         int mode = sf->mode;
1694
1695         err = dwarf_decode(ri, &cie, &fde, pc);
1696         if (err < 0)
1697                 return err;
1698
1699         sf->pc = fde.initial_location;
1700
1701         rs = &sf->rs;
1702         rs_initial = &sf->rs_initial;
1703
1704         rs->cfa_register = -1;
1705         rs_initial->cfa_register = -1;
1706
1707         rs->cfa_register = 0;
1708
1709         set_rule(rs, regnum_lr(mode), DW_WHERE_UNDEF, 0);
1710
1711         if (cie.initial_insn) {
1712                 insn_end = cie.initial_insn + cie.initial_insn_len;
1713                 err = dwarf_cfa_exec_insns(ri, cie.initial_insn,
1714                                            insn_end, &cie, sf, pc);
1715                 if (err)
1716                         return err;
1717         }
1718
1719         memcpy(rs_initial, rs, sizeof(*rs));
1720
1721         if (fde.instructions) {
1722                 insn_end = fde.instructions + fde.insn_length;
1723                 err = dwarf_cfa_exec_insns(ri, fde.instructions,
1724                                            insn_end, fde.cie, sf, pc);
1725                 if (err)
1726                         return err;
1727         }
1728
1729         pr_debug("mode: %s\n", (mode == DW_MODE_ARM32) ? "arm32" : "arm64");
1730         pr_debug("initial cfa: %#lx\n", sf->cfa);
1731
1732         user_reg_size = get_user_reg_size(mode);
1733
1734         err = def_cfa(sf, rs);
1735         if (err < 0)
1736                 return err;
1737
1738         pr_debug("pc: %#lx, lr: %#lx\n", sf->pc, sf->vregs[regnum_lr(mode)]);
1739
1740         pr_debug("sp: %#lx, fp: %#lx, fp_thumb: %#lx\n",
1741                  sf->vregs[regnum_sp(mode)],
1742                  sf->vregs[regnum_fp(mode)],
1743                  sf->vregs[ARM32_FP_THUMB]);
1744
1745         pr_debug("lr rule: %#lx/%ld (where: %u)\n",
1746                  rs->reg[regnum_lr(mode)].loc.reg,
1747                  rs->reg[regnum_lr(mode)].loc.offset,
1748                  rs->reg[regnum_lr(mode)].where);
1749
1750         pr_debug("fp rule: %#lx/%ld (where: %u)\n",
1751                  rs->reg[regnum_fp(mode)].loc.reg,
1752                  rs->reg[regnum_fp(mode)].loc.offset,
1753                  rs->reg[regnum_fp(mode)].where);
1754
1755         pr_debug("fp_thumb rule: %#lx/%ld (where: %u)\n",
1756                  rs->reg[ARM32_FP_THUMB].loc.reg,
1757                  rs->reg[ARM32_FP_THUMB].loc.offset,
1758                  rs->reg[ARM32_FP_THUMB].where);
1759
1760         pr_debug("cfa_offset: %ld (%#lx)\n",
1761                  rs->cfa_offset, rs->cfa_offset);
1762         pr_debug("cfa_register: %u\n", rs->cfa_register);
1763         pr_debug("new cfa: %#lx\n", sf->cfa);
1764
1765         for (i = 0; i < QUADD_NUM_REGS; i++) {
1766                 switch (rs->reg[i].where) {
1767                 case DW_WHERE_UNDEF:
1768                         break;
1769
1770                 case DW_WHERE_SAME:
1771                         break;
1772
1773                 case DW_WHERE_CFAREL:
1774                         addr = sf->cfa + rs->reg[i].loc.offset;
1775
1776                         if (!validate_stack_addr(addr, vma_sp, user_reg_size))
1777                                 return -QUADD_URC_SP_INCORRECT;
1778
1779                         if (mode == DW_MODE_ARM32)
1780                                 err = read_user_data((u32 __user *)addr, val);
1781                         else
1782                                 err = read_user_data((unsigned long __user *)
1783                                                      addr, val);
1784
1785                         if (err < 0)
1786                                 return err;
1787
1788                         sf->vregs[i] = val;
1789                         pr_debug("[r%d] DW_WHERE_CFAREL: new val: %#lx\n",
1790                                  i, val);
1791
1792                         break;
1793
1794                 default:
1795                         pr_err_once("[r%d] error: unsupported rule\n",
1796                                     rs->reg[i].where);
1797                         break;
1798                 }
1799         }
1800
1801         return_addr = sf->vregs[regnum_lr(mode)];
1802         pr_debug("return_addr: %#lx\n", return_addr);
1803
1804         if (!validate_pc_addr(return_addr, user_reg_size))
1805                 return -QUADD_URC_PC_INCORRECT;
1806
1807         sf->pc = return_addr;
1808         sf->vregs[regnum_sp(mode)] = sf->cfa;
1809
1810         return 0;
1811 }
1812
1813 static void
1814 unwind_backtrace(struct quadd_callchain *cc,
1815                  struct ex_region_info *ri,
1816                  struct stackframe *sf,
1817                  struct vm_area_struct *vma_sp,
1818                  struct task_struct *task)
1819 {
1820         unsigned long user_reg_size;
1821         struct ex_region_info ri_new;
1822         unsigned int unw_type = QUADD_UNW_TYPE_UT;
1823         int mode = sf->mode;
1824
1825         cc->unw_rc = QUADD_URC_FAILURE;
1826         user_reg_size = get_user_reg_size(mode);
1827
1828         while (1) {
1829                 long sp, err;
1830                 int nr_added;
1831                 struct vm_area_struct *vma_pc;
1832                 unsigned long addr, where = sf->pc;
1833                 struct mm_struct *mm = task->mm;
1834                 struct extab_info *ti;
1835
1836                 if (!mm)
1837                         break;
1838
1839                 sp = sf->vregs[regnum_sp(mode)];
1840
1841                 if (!validate_stack_addr(sp, vma_sp, user_reg_size)) {
1842                         cc->unw_rc = -QUADD_URC_SP_INCORRECT;
1843                         break;
1844                 }
1845
1846                 vma_pc = find_vma(mm, sf->pc);
1847                 if (!vma_pc)
1848                         break;
1849
1850                 ti = &ri->ex_sec[QUADD_SEC_TYPE_EH_FRAME_HDR];
1851                 addr = ti->addr;
1852
1853                 if (!is_vma_addr(addr, vma_pc, user_reg_size)) {
1854                         err = quadd_get_extabs_ehframe(vma_pc->vm_start,
1855                                                        &ri_new);
1856                         if (err) {
1857                                 cc->unw_rc = QUADD_URC_TBL_NOT_EXIST;
1858                                 break;
1859                         }
1860
1861                         ri = &ri_new;
1862                 }
1863
1864                 err = unwind_frame(ri, sf, vma_sp, &unw_type);
1865                 if (err < 0) {
1866                         cc->unw_rc = -err;
1867                         break;
1868                 }
1869
1870                 pr_debug("function at [<%08lx>] from [<%08lx>]\n",
1871                          where, sf->pc);
1872
1873                 cc->curr_sp = sf->vregs[regnum_sp(mode)];
1874                 cc->curr_fp = sf->vregs[regnum_fp(mode)];
1875                 cc->curr_pc = sf->pc;
1876
1877                 nr_added = quadd_callchain_store(cc, sf->pc, unw_type);
1878                 if (nr_added == 0)
1879                         break;
1880         }
1881 }
1882
1883 int
1884 quadd_is_ex_entry_exist_dwarf(struct pt_regs *regs,
1885                               unsigned long addr,
1886                               struct task_struct *task)
1887 {
1888         long err;
1889         unsigned char *fde_p;
1890         struct ex_region_info ri;
1891         unsigned char *hdr_start;
1892         unsigned long hdr_len, a;
1893         struct vm_area_struct *vma;
1894         struct mm_struct *mm = task->mm;
1895         struct extab_info *ti;
1896
1897         if (!regs || !mm)
1898                 return 0;
1899
1900         vma = find_vma(mm, addr);
1901         if (!vma)
1902                 return 0;
1903
1904         err = quadd_get_extabs_ehframe(vma->vm_start, &ri);
1905         if (err)
1906                 return 0;
1907
1908         ti = &ri.ex_sec[QUADD_SEC_TYPE_EH_FRAME_HDR];
1909
1910         a = ti->addr;
1911
1912         hdr_start = (unsigned char *)
1913                 ex_addr_to_mmap_addr(a, &ri, QUADD_SEC_TYPE_EH_FRAME_HDR);
1914
1915         hdr_len = ti->length;
1916
1917         fde_p = dwarf_find_fde(&ri, hdr_start, hdr_len, addr);
1918         if (!fde_p)
1919                 return 0;
1920
1921         return 1;
1922 }
1923
1924 unsigned int
1925 quadd_get_user_cc_dwarf(struct pt_regs *regs,
1926                         struct quadd_callchain *cc,
1927                         struct task_struct *task)
1928 {
1929         long err;
1930         int i, mode, nr_prev = cc->nr;
1931         unsigned long ip, lr, sp, fp, fp_thumb;
1932         struct vm_area_struct *vma, *vma_sp;
1933         struct mm_struct *mm = task->mm;
1934         struct ex_region_info ri;
1935         struct stackframe sf;
1936         struct dwarf_cpu_context *cpu_ctx = this_cpu_ptr(ctx.cpu_ctx);
1937
1938         if (!regs || !mm)
1939                 return 0;
1940
1941         if (cc->unw_rc == QUADD_URC_LEVEL_TOO_DEEP)
1942                 return nr_prev;
1943
1944         cc->unw_rc = QUADD_URC_FAILURE;
1945
1946         if (nr_prev > 0) {
1947                 ip = cc->curr_pc;
1948                 sp = cc->curr_sp;
1949                 fp = cc->curr_fp;
1950                 fp_thumb = 0;
1951                 lr = 0;
1952         } else {
1953                 ip = instruction_pointer(regs);
1954                 lr = quadd_user_link_register(regs);
1955                 sp = quadd_user_stack_pointer(regs);
1956
1957 #ifdef CONFIG_ARM64
1958                 if (compat_user_mode(regs)) {
1959                         fp = regs->compat_usr(11);
1960                         fp_thumb = regs->compat_usr(7);
1961                 } else {
1962                         fp = regs->regs[29];
1963                         fp_thumb = 0;
1964                 }
1965 #else
1966                 fp = regs->ARM_fp;
1967                 fp_thumb = regs->ARM_r7;
1968 #endif
1969         }
1970
1971 #ifdef CONFIG_ARM64
1972         if (compat_user_mode(regs))
1973                 mode = DW_MODE_ARM32;
1974         else
1975                 mode = DW_MODE_ARM64;
1976 #else
1977         mode = DW_MODE_ARM32;
1978 #endif
1979
1980
1981         pr_debug("%s: pc: %#lx, lr: %#lx\n", __func__, ip, lr);
1982         pr_debug("%s: sp: %#lx, fp: %#lx, fp_thumb: %#lx\n",
1983                  __func__, sp, fp, fp_thumb);
1984
1985         sf.vregs[regnum_lr(mode)] = lr;
1986         sf.pc = ip;
1987
1988         sf.vregs[regnum_sp(mode)] = sp;
1989         sf.vregs[regnum_fp(mode)] = fp;
1990
1991         sf.vregs[ARM32_FP_THUMB] = fp_thumb;
1992
1993         cpu_ctx->dw_word_size = (mode == DW_MODE_ARM32) ?
1994                                 sizeof(u32) : sizeof(u64);
1995
1996         sf.mode = mode;
1997         sf.cfa = 0;
1998
1999         for (i = 0; i < QUADD_NUM_REGS; i++)
2000                 set_rule(&sf.rs, i, DW_WHERE_UNDEF, 0);
2001
2002         vma = find_vma(mm, ip);
2003         if (!vma)
2004                 return 0;
2005
2006         vma_sp = find_vma(mm, sp);
2007         if (!vma_sp)
2008                 return 0;
2009
2010         err = quadd_get_extabs_ehframe(vma->vm_start, &ri);
2011         if (err) {
2012                 cc->unw_rc = QUADD_URC_TBL_NOT_EXIST;
2013                 return 0;
2014         }
2015
2016         unwind_backtrace(cc, &ri, &sf, vma_sp, task);
2017
2018         pr_debug("%s: mode: %s, cc->nr: %d --> %d\n", __func__,
2019                  (mode == DW_MODE_ARM32) ? "arm32" : "arm64",
2020                  nr_prev, cc->nr);
2021
2022         return cc->nr;
2023 }
2024
2025 int quadd_dwarf_unwind_start(void)
2026 {
2027         if (!atomic_cmpxchg(&ctx.started, 0, 1)) {
2028                 ctx.cpu_ctx = alloc_percpu(struct dwarf_cpu_context);
2029                 if (!ctx.cpu_ctx)
2030                         return -ENOMEM;
2031         }
2032
2033         return 0;
2034 }
2035
2036 void quadd_dwarf_unwind_stop(void)
2037 {
2038         if (atomic_cmpxchg(&ctx.started, 1, 0))
2039                 free_percpu(ctx.cpu_ctx);
2040 }
2041
2042 int quadd_dwarf_unwind_init(void)
2043 {
2044         atomic_set(&ctx.started, 0);
2045         return 0;
2046 }