drivers/gpu/nvgpu/gk20a/fifo_gk20a.c

   1 /*
   2  * drivers/video/tegra/host/gk20a/fifo_gk20a.c
   3  *
   4  * GK20A Graphics FIFO (gr host)
   5  *
   6  * Copyright (c) 2011-2014, NVIDIA CORPORATION.  All rights reserved.
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms and conditions of the GNU General Public License,
  10  * version 2, as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope it will be useful, but WITHOUT
  13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15  * more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along with
  18  * this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  20  */
  21 #include <linux/delay.h>
  22 #include <linux/slab.h>
  23 #include <linux/scatterlist.h>
  24 #include <trace/events/gk20a.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/nvhost.h>
  27
  28 #include "gk20a.h"
  29 #include "debug_gk20a.h"
  30 #include "hw_fifo_gk20a.h"
  31 #include "hw_pbdma_gk20a.h"
  32 #include "hw_ccsr_gk20a.h"
  33 #include "hw_ram_gk20a.h"
  34 #include "hw_proj_gk20a.h"
  35 #include "hw_top_gk20a.h"
  36 #include "hw_mc_gk20a.h"
  37 #include "hw_gr_gk20a.h"
  38
  39 static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
  40                                             u32 hw_chid, bool add,
  41                                             bool wait_for_finish);
  42 static void gk20a_fifo_handle_mmu_fault_thread(struct work_struct *work);
  43
  44 /*
  45  * Link engine IDs to MMU IDs and vice versa.
  46  */
  47
  48 static inline u32 gk20a_engine_id_to_mmu_id(u32 engine_id)
  49 {
  50         switch (engine_id) {
  51         case ENGINE_GR_GK20A:
  52                 return 0x00;
  53         case ENGINE_CE2_GK20A:
  54                 return 0x1b;
  55         default:
  56                 return ~0;
  57         }
  58 }
  59
  60 static inline u32 gk20a_mmu_id_to_engine_id(u32 engine_id)
  61 {
  62         switch (engine_id) {
  63         case 0x00:
  64                 return ENGINE_GR_GK20A;
  65         case 0x1b:
  66                 return ENGINE_CE2_GK20A;
  67         default:
  68                 return ~0;
  69         }
  70 }
  71
  72
  73 static int init_engine_info(struct fifo_gk20a *f)
  74 {
  75         struct gk20a *g = f->g;
  76         struct device *d = dev_from_gk20a(g);
  77         struct fifo_engine_info_gk20a *gr_info;
  78         const u32 gr_sw_id = ENGINE_GR_GK20A;
  79         u32 i;
  80         u32 max_info_entries = top_device_info__size_1_v();
  81
  82         gk20a_dbg_fn("");
  83
  84         /* all we really care about finding is the graphics entry    */
  85         /* especially early on in sim it probably thinks it has more */
  86         f->num_engines = 1;
  87
  88         gr_info = f->engine_info + gr_sw_id;
  89
  90         gr_info->sw_id = gr_sw_id;
  91         gr_info->name = "gr";
  92         gr_info->dev_info_id = top_device_info_type_enum_graphics_v();
  93         gr_info->mmu_fault_id = fifo_intr_mmu_fault_eng_id_graphics_v();
  94         gr_info->runlist_id = ~0;
  95         gr_info->pbdma_id   = ~0;
  96         gr_info->engine_id  = ~0;
  97
  98         for (i = 0; i < max_info_entries; i++) {
  99                 u32 table_entry = gk20a_readl(f->g, top_device_info_r(i));
 100                 u32 entry = top_device_info_entry_v(table_entry);
 101                 u32 engine_enum = top_device_info_type_enum_v(table_entry);
 102                 u32 table_entry2 = 0;
 103
 104                 if (entry == top_device_info_entry_not_valid_v())
 105                         continue;
 106
 107                 if (top_device_info_chain_v(table_entry) ==
 108                     top_device_info_chain_enable_v()) {
 109
 110                         table_entry2 = gk20a_readl(f->g,
 111                                                    top_device_info_r(++i));
 112
 113                         engine_enum = top_device_info_type_enum_v(table_entry2);
 114                 }
 115
 116                 /* we only care about GR engine here */
 117                 if (entry == top_device_info_entry_enum_v() &&
 118                     engine_enum == gr_info->dev_info_id) {
 119                         int pbdma_id;
 120                         u32 runlist_bit;
 121
 122                         gr_info->runlist_id =
 123                                 top_device_info_runlist_enum_v(table_entry);
 124                         gk20a_dbg_info("gr info: runlist_id %d", gr_info->runlist_id);
 125
 126                         gr_info->engine_id =
 127                                 top_device_info_engine_enum_v(table_entry);
 128                         gk20a_dbg_info("gr info: engine_id %d", gr_info->engine_id);
 129
 130                         runlist_bit = 1 << gr_info->runlist_id;
 131
 132                         for (pbdma_id = 0; pbdma_id < f->num_pbdma; pbdma_id++) {
 133                                 gk20a_dbg_info("gr info: pbdma_map[%d]=%d",
 134                                         pbdma_id, f->pbdma_map[pbdma_id]);
 135                                 if (f->pbdma_map[pbdma_id] & runlist_bit)
 136                                         break;
 137                         }
 138
 139                         if (pbdma_id == f->num_pbdma) {
 140                                 gk20a_err(d, "busted pbmda map");
 141                                 return -EINVAL;
 142                         }
 143                         gr_info->pbdma_id = pbdma_id;
 144
 145                         break;
 146                 }
 147         }
 148
 149         if (gr_info->runlist_id == ~0) {
 150                 gk20a_err(d, "busted device info");
 151                 return -EINVAL;
 152         }
 153
 154         return 0;
 155 }
 156
 157 void gk20a_remove_fifo_support(struct fifo_gk20a *f)
 158 {
 159         struct gk20a *g = f->g;
 160         struct device *d = dev_from_gk20a(g);
 161         struct fifo_engine_info_gk20a *engine_info;
 162         struct fifo_runlist_info_gk20a *runlist;
 163         u32 runlist_id;
 164         u32 i;
 165
 166         gk20a_dbg_fn("");
 167
 168         if (f->channel) {
 169                 int c;
 170                 for (c = 0; c < f->num_channels; c++) {
 171                         if (f->channel[c].remove_support)
 172                                 f->channel[c].remove_support(f->channel+c);
 173                 }
 174                 kfree(f->channel);
 175         }
 176         if (f->userd.gpu_va)
 177                 gk20a_gmmu_unmap(&g->mm.bar1.vm,
 178                                 f->userd.gpu_va,
 179                                 f->userd.size,
 180                                 gk20a_mem_flag_none);
 181
 182         if (f->userd.sgt)
 183                 gk20a_free_sgtable(&f->userd.sgt);
 184
 185         if (f->userd.cpuva)
 186                 dma_free_coherent(d,
 187                                 f->userd_total_size,
 188                                 f->userd.cpuva,
 189                                 f->userd.iova);
 190         f->userd.cpuva = NULL;
 191         f->userd.iova = 0;
 192
 193         engine_info = f->engine_info + ENGINE_GR_GK20A;
 194         runlist_id = engine_info->runlist_id;
 195         runlist = &f->runlist_info[runlist_id];
 196
 197         for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
 198                 if (runlist->mem[i].cpuva)
 199                         dma_free_coherent(d,
 200                                 runlist->mem[i].size,
 201                                 runlist->mem[i].cpuva,
 202                                 runlist->mem[i].iova);
 203                 runlist->mem[i].cpuva = NULL;
 204                 runlist->mem[i].iova = 0;
 205         }
 206
 207         kfree(runlist->active_channels);
 208
 209         kfree(f->runlist_info);
 210         kfree(f->pbdma_map);
 211         kfree(f->engine_info);
 212 }
 213
 214 /* reads info from hardware and fills in pbmda exception info record */
 215 static inline void get_exception_pbdma_info(
 216         struct gk20a *g,
 217         struct fifo_engine_info_gk20a *eng_info)
 218 {
 219         struct fifo_pbdma_exception_info_gk20a *e =
 220                 &eng_info->pbdma_exception_info;
 221
 222         u32 pbdma_status_r = e->status_r = gk20a_readl(g,
 223                    fifo_pbdma_status_r(eng_info->pbdma_id));
 224         e->id = fifo_pbdma_status_id_v(pbdma_status_r); /* vs. id_hw_v()? */
 225         e->id_is_chid = fifo_pbdma_status_id_type_v(pbdma_status_r) ==
 226                 fifo_pbdma_status_id_type_chid_v();
 227         e->chan_status_v  = fifo_pbdma_status_chan_status_v(pbdma_status_r);
 228         e->next_id_is_chid =
 229                 fifo_pbdma_status_next_id_type_v(pbdma_status_r) ==
 230                 fifo_pbdma_status_next_id_type_chid_v();
 231         e->next_id = fifo_pbdma_status_next_id_v(pbdma_status_r);
 232         e->chsw_in_progress =
 233                 fifo_pbdma_status_chsw_v(pbdma_status_r) ==
 234                 fifo_pbdma_status_chsw_in_progress_v();
 235 }
 236
 237 static void fifo_pbdma_exception_status(struct gk20a *g,
 238         struct fifo_engine_info_gk20a *eng_info)
 239 {
 240         struct fifo_pbdma_exception_info_gk20a *e;
 241         get_exception_pbdma_info(g, eng_info);
 242         e = &eng_info->pbdma_exception_info;
 243
 244         gk20a_dbg_fn("pbdma_id %d, "
 245                       "id_type %s, id %d, chan_status %d, "
 246                       "next_id_type %s, next_id %d, "
 247                       "chsw_in_progress %d",
 248                       eng_info->pbdma_id,
 249                       e->id_is_chid ? "chid" : "tsgid", e->id, e->chan_status_v,
 250                       e->next_id_is_chid ? "chid" : "tsgid", e->next_id,
 251                       e->chsw_in_progress);
 252 }
 253
 254 /* reads info from hardware and fills in pbmda exception info record */
 255 static inline void get_exception_engine_info(
 256         struct gk20a *g,
 257         struct fifo_engine_info_gk20a *eng_info)
 258 {
 259         struct fifo_engine_exception_info_gk20a *e =
 260                 &eng_info->engine_exception_info;
 261         u32 engine_status_r = e->status_r =
 262                 gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id));
 263         e->id = fifo_engine_status_id_v(engine_status_r); /* vs. id_hw_v()? */
 264         e->id_is_chid = fifo_engine_status_id_type_v(engine_status_r) ==
 265                 fifo_engine_status_id_type_chid_v();
 266         e->ctx_status_v = fifo_engine_status_ctx_status_v(engine_status_r);
 267         e->faulted =
 268                 fifo_engine_status_faulted_v(engine_status_r) ==
 269                 fifo_engine_status_faulted_true_v();
 270         e->idle =
 271                 fifo_engine_status_engine_v(engine_status_r) ==
 272                 fifo_engine_status_engine_idle_v();
 273         e->ctxsw_in_progress =
 274                 fifo_engine_status_ctxsw_v(engine_status_r) ==
 275                 fifo_engine_status_ctxsw_in_progress_v();
 276 }
 277
 278 static void fifo_engine_exception_status(struct gk20a *g,
 279                                struct fifo_engine_info_gk20a *eng_info)
 280 {
 281         struct fifo_engine_exception_info_gk20a *e;
 282         get_exception_engine_info(g, eng_info);
 283         e = &eng_info->engine_exception_info;
 284
 285         gk20a_dbg_fn("engine_id %d, id_type %s, id %d, ctx_status %d, "
 286                       "faulted %d, idle %d, ctxsw_in_progress %d, ",
 287                       eng_info->engine_id, e->id_is_chid ? "chid" : "tsgid",
 288                       e->id, e->ctx_status_v,
 289                       e->faulted, e->idle,  e->ctxsw_in_progress);
 290 }
 291
 292 static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
 293 {
 294         struct fifo_engine_info_gk20a *engine_info;
 295         struct fifo_runlist_info_gk20a *runlist;
 296         struct device *d = dev_from_gk20a(g);
 297         u32 runlist_id;
 298         u32 i;
 299         u64 runlist_size;
 300
 301         gk20a_dbg_fn("");
 302
 303         f->max_runlists = fifo_eng_runlist_base__size_1_v();
 304         f->runlist_info = kzalloc(sizeof(struct fifo_runlist_info_gk20a) *
 305                                   f->max_runlists, GFP_KERNEL);
 306         if (!f->runlist_info)
 307                 goto clean_up;
 308
 309         engine_info = f->engine_info + ENGINE_GR_GK20A;
 310         runlist_id = engine_info->runlist_id;
 311         runlist = &f->runlist_info[runlist_id];
 312
 313         runlist->active_channels =
 314                 kzalloc(DIV_ROUND_UP(f->num_channels, BITS_PER_BYTE),
 315                         GFP_KERNEL);
 316         if (!runlist->active_channels)
 317                 goto clean_up_runlist_info;
 318
 319         runlist_size  = ram_rl_entry_size_v() * f->num_channels;
 320         for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
 321                 dma_addr_t iova;
 322
 323                 runlist->mem[i].cpuva =
 324                         dma_alloc_coherent(d,
 325                                         runlist_size,
 326                                         &iova,
 327                                         GFP_KERNEL);
 328                 if (!runlist->mem[i].cpuva) {
 329                         dev_err(d, "memory allocation failed\n");
 330                         goto clean_up_runlist;
 331                 }
 332                 runlist->mem[i].iova = iova;
 333                 runlist->mem[i].size = runlist_size;
 334         }
 335         mutex_init(&runlist->mutex);
 336         init_waitqueue_head(&runlist->runlist_wq);
 337
 338         /* None of buffers is pinned if this value doesn't change.
 339             Otherwise, one of them (cur_buffer) must have been pinned. */
 340         runlist->cur_buffer = MAX_RUNLIST_BUFFERS;
 341
 342         gk20a_dbg_fn("done");
 343         return 0;
 344
 345 clean_up_runlist:
 346         for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
 347                 if (runlist->mem[i].cpuva)
 348                         dma_free_coherent(d,
 349                                 runlist->mem[i].size,
 350                                 runlist->mem[i].cpuva,
 351                                 runlist->mem[i].iova);
 352                 runlist->mem[i].cpuva = NULL;
 353                 runlist->mem[i].iova = 0;
 354         }
 355
 356         kfree(runlist->active_channels);
 357         runlist->active_channels = NULL;
 358
 359 clean_up_runlist_info:
 360         kfree(f->runlist_info);
 361         f->runlist_info = NULL;
 362
 363 clean_up:
 364         gk20a_dbg_fn("fail");
 365         return -ENOMEM;
 366 }
 367
 368 #define GRFIFO_TIMEOUT_CHECK_PERIOD_US 100000
 369
 370 int gk20a_init_fifo_reset_enable_hw(struct gk20a *g)
 371 {
 372         u32 intr_stall;
 373         u32 mask;
 374         u32 timeout;
 375         int i;
 376
 377         gk20a_dbg_fn("");
 378         /* enable pmc pfifo */
 379         gk20a_reset(g, mc_enable_pfifo_enabled_f()
 380                         | mc_enable_ce2_enabled_f());
 381
 382         /* enable pbdma */
 383         mask = 0;
 384         for (i = 0; i < proj_host_num_pbdma_v(); ++i)
 385                 mask |= mc_enable_pb_sel_f(mc_enable_pb_0_enabled_v(), i);
 386         gk20a_writel(g, mc_enable_pb_r(), mask);
 387
 388         /* enable pfifo interrupt */
 389         gk20a_writel(g, fifo_intr_0_r(), 0xFFFFFFFF);
 390         gk20a_writel(g, fifo_intr_en_0_r(), 0x7FFFFFFF);
 391         gk20a_writel(g, fifo_intr_en_1_r(), 0x80000000);
 392
 393         /* enable pbdma interrupt */
 394         mask = 0;
 395         for (i = 0; i < proj_host_num_pbdma_v(); i++) {
 396                 intr_stall = gk20a_readl(g, pbdma_intr_stall_r(i));
 397                 intr_stall &= ~pbdma_intr_stall_lbreq_enabled_f();
 398                 gk20a_writel(g, pbdma_intr_stall_r(i), intr_stall);
 399                 gk20a_writel(g, pbdma_intr_0_r(i), 0xFFFFFFFF);
 400                 gk20a_writel(g, pbdma_intr_en_0_r(i),
 401                         ~pbdma_intr_en_0_lbreq_enabled_f());
 402                 gk20a_writel(g, pbdma_intr_1_r(i), 0xFFFFFFFF);
 403                 gk20a_writel(g, pbdma_intr_en_1_r(i),
 404                         ~pbdma_intr_en_0_lbreq_enabled_f());
 405         }
 406
 407         /* TBD: apply overrides */
 408
 409         /* TBD: BLCG prod */
 410
 411         /* reset runlist interrupts */
 412         gk20a_writel(g, fifo_intr_runlist_r(), ~0);
 413
 414         /* TBD: do we need those? */
 415         timeout = gk20a_readl(g, fifo_fb_timeout_r());
 416         timeout = set_field(timeout, fifo_fb_timeout_period_m(),
 417                         fifo_fb_timeout_period_max_f());
 418         gk20a_writel(g, fifo_fb_timeout_r(), timeout);
 419
 420         for (i = 0; i < pbdma_timeout__size_1_v(); i++) {
 421                 timeout = gk20a_readl(g, pbdma_timeout_r(i));
 422                 timeout = set_field(timeout, pbdma_timeout_period_m(),
 423                                     pbdma_timeout_period_max_f());
 424                 gk20a_writel(g, pbdma_timeout_r(i), timeout);
 425         }
 426
 427         if (tegra_platform_is_silicon()) {
 428                 timeout = gk20a_readl(g, fifo_pb_timeout_r());
 429                 timeout &= ~fifo_pb_timeout_detection_enabled_f();
 430                 gk20a_writel(g, fifo_pb_timeout_r(), timeout);
 431         }
 432
 433         timeout = GRFIFO_TIMEOUT_CHECK_PERIOD_US |
 434                         fifo_eng_timeout_detection_enabled_f();
 435         gk20a_writel(g, fifo_eng_timeout_r(), timeout);
 436
 437         gk20a_dbg_fn("done");
 438
 439         return 0;
 440 }
 441
 442 static void gk20a_init_fifo_pbdma_intr_descs(struct fifo_gk20a *f)
 443 {
 444         /* These are all errors which indicate something really wrong
 445          * going on in the device. */
 446         f->intr.pbdma.device_fatal_0 =
 447                 pbdma_intr_0_memreq_pending_f() |
 448                 pbdma_intr_0_memack_timeout_pending_f() |
 449                 pbdma_intr_0_memack_extra_pending_f() |
 450                 pbdma_intr_0_memdat_timeout_pending_f() |
 451                 pbdma_intr_0_memdat_extra_pending_f() |
 452                 pbdma_intr_0_memflush_pending_f() |
 453                 pbdma_intr_0_memop_pending_f() |
 454                 pbdma_intr_0_lbconnect_pending_f() |
 455                 pbdma_intr_0_lback_timeout_pending_f() |
 456                 pbdma_intr_0_lback_extra_pending_f() |
 457                 pbdma_intr_0_lbdat_timeout_pending_f() |
 458                 pbdma_intr_0_lbdat_extra_pending_f() |
 459                 pbdma_intr_0_xbarconnect_pending_f() |
 460                 pbdma_intr_0_pri_pending_f();
 461
 462         /* These are data parsing, framing errors or others which can be
 463          * recovered from with intervention... or just resetting the
 464          * channel. */
 465         f->intr.pbdma.channel_fatal_0 =
 466                 pbdma_intr_0_gpfifo_pending_f() |
 467                 pbdma_intr_0_gpptr_pending_f() |
 468                 pbdma_intr_0_gpentry_pending_f() |
 469                 pbdma_intr_0_gpcrc_pending_f() |
 470                 pbdma_intr_0_pbptr_pending_f() |
 471                 pbdma_intr_0_pbentry_pending_f() |
 472                 pbdma_intr_0_pbcrc_pending_f() |
 473                 pbdma_intr_0_method_pending_f() |
 474                 pbdma_intr_0_methodcrc_pending_f() |
 475                 pbdma_intr_0_pbseg_pending_f() |
 476                 pbdma_intr_0_signature_pending_f();
 477
 478         /* Can be used for sw-methods, or represents
 479          * a recoverable timeout. */
 480         f->intr.pbdma.restartable_0 =
 481                 pbdma_intr_0_device_pending_f() |
 482                 pbdma_intr_0_acquire_pending_f();
 483 }
 484
 485 static int gk20a_init_fifo_setup_sw(struct gk20a *g)
 486 {
 487         struct fifo_gk20a *f = &g->fifo;
 488         struct device *d = dev_from_gk20a(g);
 489         int chid, i, err = 0;
 490         dma_addr_t iova;
 491
 492         gk20a_dbg_fn("");
 493
 494         if (f->sw_ready) {
 495                 gk20a_dbg_fn("skip init");
 496                 return 0;
 497         }
 498
 499         f->g = g;
 500
 501         INIT_WORK(&f->fault_restore_thread,
 502                   gk20a_fifo_handle_mmu_fault_thread);
 503         mutex_init(&f->intr.isr.mutex);
 504         gk20a_init_fifo_pbdma_intr_descs(f); /* just filling in data/tables */
 505
 506         f->num_channels = ccsr_channel__size_1_v();
 507         f->num_pbdma = proj_host_num_pbdma_v();
 508         f->max_engines = ENGINE_INVAL_GK20A;
 509
 510         f->userd_entry_size = 1 << ram_userd_base_shift_v();
 511         f->userd_total_size = f->userd_entry_size * f->num_channels;
 512
 513         f->userd.cpuva = dma_alloc_coherent(d,
 514                                         f->userd_total_size,
 515                                         &iova,
 516                                         GFP_KERNEL);
 517         if (!f->userd.cpuva) {
 518                 dev_err(d, "memory allocation failed\n");
 519                 goto clean_up;
 520         }
 521
 522         f->userd.iova = iova;
 523         err = gk20a_get_sgtable(d, &f->userd.sgt,
 524                                 f->userd.cpuva, f->userd.iova,
 525                                 f->userd_total_size);
 526         if (err) {
 527                 dev_err(d, "failed to create sg table\n");
 528                 goto clean_up;
 529         }
 530
 531         /* bar1 va */
 532         f->userd.gpu_va = gk20a_gmmu_map(&g->mm.bar1.vm,
 533                                         &f->userd.sgt,
 534                                         f->userd_total_size,
 535                                         0, /* flags */
 536                                         gk20a_mem_flag_none);
 537         if (!f->userd.gpu_va) {
 538                 dev_err(d, "gmmu mapping failed\n");
 539                 goto clean_up;
 540         }
 541
 542         gk20a_dbg(gpu_dbg_map, "userd bar1 va = 0x%llx", f->userd.gpu_va);
 543
 544         f->userd.size = f->userd_total_size;
 545
 546         f->channel = kzalloc(f->num_channels * sizeof(*f->channel),
 547                                 GFP_KERNEL);
 548         f->pbdma_map = kzalloc(f->num_pbdma * sizeof(*f->pbdma_map),
 549                                 GFP_KERNEL);
 550         f->engine_info = kzalloc(f->max_engines * sizeof(*f->engine_info),
 551                                 GFP_KERNEL);
 552
 553         if (!(f->channel && f->pbdma_map && f->engine_info)) {
 554                 err = -ENOMEM;
 555                 goto clean_up;
 556         }
 557
 558         /* pbdma map needs to be in place before calling engine info init */
 559         for (i = 0; i < f->num_pbdma; ++i)
 560                 f->pbdma_map[i] = gk20a_readl(g, fifo_pbdma_map_r(i));
 561
 562         init_engine_info(f);
 563
 564         init_runlist(g, f);
 565
 566         for (chid = 0; chid < f->num_channels; chid++) {
 567                 f->channel[chid].userd_cpu_va =
 568                         f->userd.cpuva + chid * f->userd_entry_size;
 569                 f->channel[chid].userd_iova =
 570                         NV_MC_SMMU_VADDR_TRANSLATE(f->userd.iova)
 571                                 + chid * f->userd_entry_size;
 572                 f->channel[chid].userd_gpu_va =
 573                         f->userd.gpu_va + chid * f->userd_entry_size;
 574
 575                 gk20a_init_channel_support(g, chid);
 576         }
 577         mutex_init(&f->ch_inuse_mutex);
 578
 579         f->remove_support = gk20a_remove_fifo_support;
 580
 581         f->deferred_reset_pending = false;
 582         mutex_init(&f->deferred_reset_mutex);
 583
 584         f->sw_ready = true;
 585
 586         gk20a_dbg_fn("done");
 587         return 0;
 588
 589 clean_up:
 590         gk20a_dbg_fn("fail");
 591         if (f->userd.gpu_va)
 592                 gk20a_gmmu_unmap(&g->mm.bar1.vm,
 593                                         f->userd.gpu_va,
 594                                         f->userd.size,
 595                                         gk20a_mem_flag_none);
 596         if (f->userd.sgt)
 597                 gk20a_free_sgtable(&f->userd.sgt);
 598         if (f->userd.cpuva)
 599                 dma_free_coherent(d,
 600                                 f->userd_total_size,
 601                                 f->userd.cpuva,
 602                                 f->userd.iova);
 603         f->userd.cpuva = NULL;
 604         f->userd.iova = 0;
 605
 606         memset(&f->userd, 0, sizeof(struct userd_desc));
 607
 608         kfree(f->channel);
 609         f->channel = NULL;
 610         kfree(f->pbdma_map);
 611         f->pbdma_map = NULL;
 612         kfree(f->engine_info);
 613         f->engine_info = NULL;
 614
 615         return err;
 616 }
 617
 618 static void gk20a_fifo_handle_runlist_event(struct gk20a *g)
 619 {
 620         struct fifo_gk20a *f = &g->fifo;
 621         struct fifo_runlist_info_gk20a *runlist;
 622         unsigned long runlist_event;
 623         u32 runlist_id;
 624
 625         runlist_event = gk20a_readl(g, fifo_intr_runlist_r());
 626         gk20a_writel(g, fifo_intr_runlist_r(), runlist_event);
 627
 628         for_each_set_bit(runlist_id, &runlist_event, f->max_runlists) {
 629                 runlist = &f->runlist_info[runlist_id];
 630                 wake_up(&runlist->runlist_wq);
 631         }
 632
 633 }
 634
 635 static int gk20a_init_fifo_setup_hw(struct gk20a *g)
 636 {
 637         struct fifo_gk20a *f = &g->fifo;
 638
 639         gk20a_dbg_fn("");
 640
 641         /* test write, read through bar1 @ userd region before
 642          * turning on the snooping */
 643         {
 644                 struct fifo_gk20a *f = &g->fifo;
 645                 u32 v, v1 = 0x33, v2 = 0x55;
 646
 647                 u32 bar1_vaddr = f->userd.gpu_va;
 648                 volatile u32 *cpu_vaddr = f->userd.cpuva;
 649
 650                 gk20a_dbg_info("test bar1 @ vaddr 0x%x",
 651                            bar1_vaddr);
 652
 653                 v = gk20a_bar1_readl(g, bar1_vaddr);
 654
 655                 *cpu_vaddr = v1;
 656                 smp_mb();
 657
 658                 if (v1 != gk20a_bar1_readl(g, bar1_vaddr)) {
 659                         gk20a_err(dev_from_gk20a(g), "bar1 broken @ gk20a!");
 660                         return -EINVAL;
 661                 }
 662
 663                 gk20a_bar1_writel(g, bar1_vaddr, v2);
 664
 665                 if (v2 != gk20a_bar1_readl(g, bar1_vaddr)) {
 666                         gk20a_err(dev_from_gk20a(g), "bar1 broken @ gk20a!");
 667                         return -EINVAL;
 668                 }
 669
 670                 /* is it visible to the cpu? */
 671                 if (*cpu_vaddr != v2) {
 672                         gk20a_err(dev_from_gk20a(g),
 673                                 "cpu didn't see bar1 write @ %p!",
 674                                 cpu_vaddr);
 675                 }
 676
 677                 /* put it back */
 678                 gk20a_bar1_writel(g, bar1_vaddr, v);
 679         }
 680
 681         /*XXX all manner of flushes and caching worries, etc */
 682
 683         /* set the base for the userd region now */
 684         gk20a_writel(g, fifo_bar1_base_r(),
 685                         fifo_bar1_base_ptr_f(f->userd.gpu_va >> 12) |
 686                         fifo_bar1_base_valid_true_f());
 687
 688         gk20a_dbg_fn("done");
 689
 690         return 0;
 691 }
 692
 693 int gk20a_init_fifo_support(struct gk20a *g)
 694 {
 695         u32 err;
 696
 697         err = gk20a_init_fifo_setup_sw(g);
 698         if (err)
 699                 return err;
 700
 701         err = gk20a_init_fifo_setup_hw(g);
 702         if (err)
 703                 return err;
 704
 705         return err;
 706 }
 707
 708 static struct channel_gk20a *
 709 channel_from_inst_ptr(struct fifo_gk20a *f, u64 inst_ptr)
 710 {
 711         int ci;
 712         if (unlikely(!f->channel))
 713                 return NULL;
 714         for (ci = 0; ci < f->num_channels; ci++) {
 715                 struct channel_gk20a *c = f->channel+ci;
 716                 if (c->inst_block.cpuva &&
 717                     (inst_ptr == c->inst_block.cpu_pa))
 718                         return f->channel+ci;
 719         }
 720         return NULL;
 721 }
 722
 723 /* fault info/descriptions.
 724  * tbd: move to setup
 725  *  */
 726 static const char * const fault_type_descs[] = {
 727          "pde", /*fifo_intr_mmu_fault_info_type_pde_v() == 0 */
 728          "pde size",
 729          "pte",
 730          "va limit viol",
 731          "unbound inst",
 732          "priv viol",
 733          "ro viol",
 734          "wo viol",
 735          "pitch mask",
 736          "work creation",
 737          "bad aperture",
 738          "compression failure",
 739          "bad kind",
 740          "region viol",
 741          "dual ptes",
 742          "poisoned",
 743 };
 744 /* engine descriptions */
 745 static const char * const engine_subid_descs[] = {
 746         "gpc",
 747         "hub",
 748 };
 749
 750 static const char * const hub_client_descs[] = {
 751         "vip", "ce0", "ce1", "dniso", "fe", "fecs", "host", "host cpu",
 752         "host cpu nb", "iso", "mmu", "mspdec", "msppp", "msvld",
 753         "niso", "p2p", "pd", "perf", "pmu", "raster twod", "scc",
 754         "scc nb", "sec", "ssync", "gr copy", "ce2", "xv", "mmu nb",
 755         "msenc", "d falcon", "sked", "a falcon", "n/a",
 756 };
 757
 758 static const char * const gpc_client_descs[] = {
 759         "l1 0", "t1 0", "pe 0",
 760         "l1 1", "t1 1", "pe 1",
 761         "l1 2", "t1 2", "pe 2",
 762         "l1 3", "t1 3", "pe 3",
 763         "rast", "gcc", "gpccs",
 764         "prop 0", "prop 1", "prop 2", "prop 3",
 765         "l1 4", "t1 4", "pe 4",
 766         "l1 5", "t1 5", "pe 5",
 767         "l1 6", "t1 6", "pe 6",
 768         "l1 7", "t1 7", "pe 7",
 769         "gpm",
 770         "ltp utlb 0", "ltp utlb 1", "ltp utlb 2", "ltp utlb 3",
 771         "rgg utlb",
 772 };
 773
 774 /* reads info from hardware and fills in mmu fault info record */
 775 static inline void get_exception_mmu_fault_info(
 776         struct gk20a *g, u32 engine_id,
 777         struct fifo_mmu_fault_info_gk20a *f)
 778 {
 779         u32 fault_info_v;
 780
 781         gk20a_dbg_fn("engine_id %d", engine_id);
 782
 783         memset(f, 0, sizeof(*f));
 784
 785         f->fault_info_v = fault_info_v = gk20a_readl(g,
 786              fifo_intr_mmu_fault_info_r(engine_id));
 787         f->fault_type_v =
 788                 fifo_intr_mmu_fault_info_type_v(fault_info_v);
 789         f->engine_subid_v =
 790                 fifo_intr_mmu_fault_info_engine_subid_v(fault_info_v);
 791         f->client_v = fifo_intr_mmu_fault_info_client_v(fault_info_v);
 792
 793         BUG_ON(f->fault_type_v >= ARRAY_SIZE(fault_type_descs));
 794         f->fault_type_desc =  fault_type_descs[f->fault_type_v];
 795
 796         BUG_ON(f->engine_subid_v >= ARRAY_SIZE(engine_subid_descs));
 797         f->engine_subid_desc = engine_subid_descs[f->engine_subid_v];
 798
 799         if (f->engine_subid_v ==
 800             fifo_intr_mmu_fault_info_engine_subid_hub_v()) {
 801
 802                 BUG_ON(f->client_v >= ARRAY_SIZE(hub_client_descs));
 803                 f->client_desc = hub_client_descs[f->client_v];
 804         } else if (f->engine_subid_v ==
 805                    fifo_intr_mmu_fault_info_engine_subid_gpc_v()) {
 806                 BUG_ON(f->client_v >= ARRAY_SIZE(gpc_client_descs));
 807                 f->client_desc = gpc_client_descs[f->client_v];
 808         } else {
 809                 BUG_ON(1);
 810         }
 811
 812         f->fault_hi_v = gk20a_readl(g, fifo_intr_mmu_fault_hi_r(engine_id));
 813         f->fault_lo_v = gk20a_readl(g, fifo_intr_mmu_fault_lo_r(engine_id));
 814         /* note:ignoring aperture on gk20a... */
 815         f->inst_ptr = fifo_intr_mmu_fault_inst_ptr_v(
 816                  gk20a_readl(g, fifo_intr_mmu_fault_inst_r(engine_id)));
 817         /* note: inst_ptr is a 40b phys addr.  */
 818         f->inst_ptr <<= fifo_intr_mmu_fault_inst_ptr_align_shift_v();
 819 }
 820
 821 static void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id)
 822 {
 823         gk20a_dbg_fn("");
 824
 825         if (engine_id == top_device_info_type_enum_graphics_v()) {
 826                 /* resetting engine using mc_enable_r() is not enough,
 827                  * we do full init sequence */
 828                 gk20a_gr_reset(g);
 829         }
 830         if (engine_id == top_device_info_type_enum_copy0_v())
 831                 gk20a_reset(g, mc_enable_ce2_m());
 832 }
 833
 834 static void gk20a_fifo_handle_mmu_fault_thread(struct work_struct *work)
 835 {
 836         struct fifo_gk20a *f = container_of(work, struct fifo_gk20a,
 837                                             fault_restore_thread);
 838         struct gk20a *g = f->g;
 839         int i;
 840
 841         /* It is safe to enable ELPG again. */
 842         gk20a_pmu_enable_elpg(g);
 843
 844         /* Restore the runlist */
 845         for (i = 0; i < g->fifo.max_runlists; i++)
 846                 gk20a_fifo_update_runlist_locked(g, i, ~0, true, true);
 847
 848         /* unlock all runlists */
 849         for (i = 0; i < g->fifo.max_runlists; i++)
 850                 mutex_unlock(&g->fifo.runlist_info[i].mutex);
 851
 852 }
 853
 854 static void gk20a_fifo_handle_chsw_fault(struct gk20a *g)
 855 {
 856         u32 intr;
 857
 858         intr = gk20a_readl(g, fifo_intr_chsw_error_r());
 859         gk20a_err(dev_from_gk20a(g), "chsw: %08x\n", intr);
 860         gk20a_fecs_dump_falcon_stats(g);
 861         gk20a_writel(g, fifo_intr_chsw_error_r(), intr);
 862 }
 863
 864 static void gk20a_fifo_handle_dropped_mmu_fault(struct gk20a *g)
 865 {
 866         struct device *dev = dev_from_gk20a(g);
 867         u32 fault_id = gk20a_readl(g, fifo_intr_mmu_fault_id_r());
 868         gk20a_err(dev, "dropped mmu fault (0x%08x)", fault_id);
 869 }
 870
 871 static bool gk20a_fifo_should_defer_engine_reset(struct gk20a *g, u32 engine_id,
 872                 struct fifo_mmu_fault_info_gk20a *f, bool fake_fault)
 873 {
 874         /* channel recovery is only deferred if an sm debugger
 875            is attached and has MMU debug mode is enabled */
 876         if (!gk20a_gr_sm_debugger_attached(g) ||
 877             !gk20a_mm_mmu_debug_mode_enabled(g))
 878                 return false;
 879
 880         /* if this fault is fake (due to RC recovery), don't defer recovery */
 881         if (fake_fault)
 882                 return false;
 883
 884         if (engine_id != ENGINE_GR_GK20A ||
 885             f->engine_subid_v != fifo_intr_mmu_fault_info_engine_subid_gpc_v())
 886                 return false;
 887
 888         return true;
 889 }
 890
 891 void fifo_gk20a_finish_mmu_fault_handling(struct gk20a *g,
 892                 unsigned long fault_id) {
 893         u32 engine_mmu_id;
 894         int i;
 895
 896         /* reset engines */
 897         for_each_set_bit(engine_mmu_id, &fault_id, 32) {
 898                 u32 engine_id = gk20a_mmu_id_to_engine_id(engine_mmu_id);
 899                 if (engine_id != ~0)
 900                         gk20a_fifo_reset_engine(g, engine_id);
 901         }
 902
 903         /* CLEAR the runlists. Do not wait for runlist to start as
 904          * some engines may not be available right now */
 905         for (i = 0; i < g->fifo.max_runlists; i++)
 906                 gk20a_fifo_update_runlist_locked(g, i, ~0, false, false);
 907
 908         /* clear interrupt */
 909         gk20a_writel(g, fifo_intr_mmu_fault_id_r(), fault_id);
 910
 911         /* resume scheduler */
 912         gk20a_writel(g, fifo_error_sched_disable_r(),
 913                      gk20a_readl(g, fifo_error_sched_disable_r()));
 914
 915         /* Spawn a work to enable PMU and restore runlists */
 916         schedule_work(&g->fifo.fault_restore_thread);
 917 }
 918
 919 static bool gk20a_fifo_set_ctx_mmu_error(struct gk20a *g,
 920                 struct channel_gk20a *ch) {
 921         bool verbose = true;
 922         if (!ch)
 923                 return verbose;
 924
 925         gk20a_err(dev_from_gk20a(g),
 926                 "channel %d generated a mmu fault",
 927                 ch->hw_chid);
 928         if (ch->error_notifier) {
 929                 u32 err = ch->error_notifier->info32;
 930                 if (ch->error_notifier->status == 0xffff) {
 931                         /* If error code is already set, this mmu fault
 932                          * was triggered as part of recovery from other
 933                          * error condition.
 934                          * Don't overwrite error flag. */
 935                         /* Fifo timeout debug spew is controlled by user */
 936                         if (err == NVHOST_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT)
 937                                 verbose = ch->timeout_debug_dump;
 938                 } else {
 939                         gk20a_set_error_notifier(ch,
 940                                 NVHOST_CHANNEL_FIFO_ERROR_MMU_ERR_FLT);
 941                 }
 942         }
 943         /* mark channel as faulted */
 944         ch->has_timedout = true;
 945         wmb();
 946         /* unblock pending waits */
 947         wake_up(&ch->semaphore_wq);
 948         wake_up(&ch->notifier_wq);
 949         wake_up(&ch->submit_wq);
 950         return verbose;
 951 }
 952
 953
 954 static bool gk20a_fifo_handle_mmu_fault(struct gk20a *g)
 955 {
 956         bool fake_fault;
 957         unsigned long fault_id;
 958         unsigned long engine_mmu_id;
 959         int i;
 960         bool verbose = true;
 961         gk20a_dbg_fn("");
 962
 963         g->fifo.deferred_reset_pending = false;
 964
 965         /* Disable ELPG */
 966         gk20a_pmu_disable_elpg(g);
 967
 968         /* If we have recovery in progress, MMU fault id is invalid */
 969         if (g->fifo.mmu_fault_engines) {
 970                 fault_id = g->fifo.mmu_fault_engines;
 971                 g->fifo.mmu_fault_engines = 0;
 972                 fake_fault = true;
 973         } else {
 974                 fault_id = gk20a_readl(g, fifo_intr_mmu_fault_id_r());
 975                 fake_fault = false;
 976                 gk20a_debug_dump(g->dev);
 977         }
 978
 979         /* lock all runlists. Note that locks are are released in
 980          * gk20a_fifo_handle_mmu_fault_thread() */
 981         for (i = 0; i < g->fifo.max_runlists; i++)
 982                 mutex_lock(&g->fifo.runlist_info[i].mutex);
 983
 984         /* go through all faulted engines */
 985         for_each_set_bit(engine_mmu_id, &fault_id, 32) {
 986                 /* bits in fifo_intr_mmu_fault_id_r do not correspond 1:1 to
 987                  * engines. Convert engine_mmu_id to engine_id */
 988                 u32 engine_id = gk20a_mmu_id_to_engine_id(engine_mmu_id);
 989                 struct fifo_runlist_info_gk20a *runlist = g->fifo.runlist_info;
 990                 struct fifo_mmu_fault_info_gk20a f;
 991                 struct channel_gk20a *ch = NULL;
 992
 993                 get_exception_mmu_fault_info(g, engine_mmu_id, &f);
 994                 trace_gk20a_mmu_fault(f.fault_hi_v,
 995                                       f.fault_lo_v,
 996                                       f.fault_info_v,
 997                                       f.inst_ptr,
 998                                       engine_id,
 999                                       f.engine_subid_desc,
1000                                       f.client_desc,
1001                                       f.fault_type_desc);
1002                 gk20a_err(dev_from_gk20a(g), "mmu fault on engine %d, "
1003                            "engine subid %d (%s), client %d (%s), "
1004                            "addr 0x%08x:0x%08x, type %d (%s), info 0x%08x,"
1005                            "inst_ptr 0x%llx\n",
1006                            engine_id,
1007                            f.engine_subid_v, f.engine_subid_desc,
1008                            f.client_v, f.client_desc,
1009                            f.fault_hi_v, f.fault_lo_v,
1010                            f.fault_type_v, f.fault_type_desc,
1011                            f.fault_info_v, f.inst_ptr);
1012
1013                 /* get the channel */
1014                 if (fake_fault) {
1015                         /* read and parse engine status */
1016                         u32 status = gk20a_readl(g,
1017                                 fifo_engine_status_r(engine_id));
1018                         u32 ctx_status =
1019                                 fifo_engine_status_ctx_status_v(status);
1020                         bool type_ch = fifo_pbdma_status_id_type_v(status) ==
1021                                 fifo_pbdma_status_id_type_chid_v();
1022
1023                         /* use next_id if context load is failing */
1024                         u32 id = (ctx_status ==
1025                                 fifo_engine_status_ctx_status_ctxsw_load_v()) ?
1026                                 fifo_engine_status_next_id_v(status) :
1027                                 fifo_engine_status_id_v(status);
1028
1029                         if (type_ch) {
1030                                 ch = g->fifo.channel + id;
1031                         } else {
1032                                 gk20a_err(dev_from_gk20a(g), "non-chid type not supported");
1033                                 WARN_ON(1);
1034                         }
1035                 } else {
1036                         /* read channel based on instruction pointer */
1037                         ch = channel_from_inst_ptr(&g->fifo, f.inst_ptr);
1038                 }
1039
1040                 if (ch) {
1041                         if (ch->in_use) {
1042                                 /* disable the channel from hw and increment
1043                                  * syncpoints */
1044                                 gk20a_disable_channel_no_update(ch);
1045
1046                                 /* remove the channel from runlist */
1047                                 clear_bit(ch->hw_chid,
1048                                           runlist->active_channels);
1049                         }
1050
1051                         /* check if engine reset should be deferred */
1052                         if (gk20a_fifo_should_defer_engine_reset(g, engine_id, &f, fake_fault)) {
1053                                 g->fifo.mmu_fault_engines = fault_id;
1054
1055                                 /* handled during channel free */
1056                                 g->fifo.deferred_reset_pending = true;
1057                         } else
1058                                 verbose = gk20a_fifo_set_ctx_mmu_error(g, ch);
1059
1060                 } else if (f.inst_ptr ==
1061                                 g->mm.bar1.inst_block.cpu_pa) {
1062                         gk20a_err(dev_from_gk20a(g), "mmu fault from bar1");
1063                 } else if (f.inst_ptr ==
1064                                 g->mm.pmu.inst_block.cpu_pa) {
1065                         gk20a_err(dev_from_gk20a(g), "mmu fault from pmu");
1066                 } else
1067                         gk20a_err(dev_from_gk20a(g), "couldn't locate channel for mmu fault");
1068         }
1069
1070         if (g->fifo.deferred_reset_pending) {
1071                 gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, "sm debugger attached,"
1072                            " deferring channel recovery to channel free");
1073                 /* clear interrupt */
1074                 gk20a_writel(g, fifo_intr_mmu_fault_id_r(), fault_id);
1075                 return verbose;
1076         }
1077
1078         /* resetting the engines and clearing the runlists is done in
1079            a separate function to allow deferred reset. */
1080         fifo_gk20a_finish_mmu_fault_handling(g, fault_id);
1081         return verbose;
1082 }
1083
1084 static void gk20a_fifo_get_faulty_channel(struct gk20a *g, int engine_id,
1085                                           u32 *chid, bool *type_ch)
1086 {
1087         u32 status = gk20a_readl(g, fifo_engine_status_r(engine_id));
1088         u32 ctx_status = fifo_engine_status_ctx_status_v(status);
1089
1090         *type_ch = fifo_pbdma_status_id_type_v(status) ==
1091                 fifo_pbdma_status_id_type_chid_v();
1092         /* use next_id if context load is failing */
1093         *chid = (ctx_status ==
1094                 fifo_engine_status_ctx_status_ctxsw_load_v()) ?
1095                 fifo_engine_status_next_id_v(status) :
1096                 fifo_engine_status_id_v(status);
1097 }
1098
1099 static void gk20a_fifo_trigger_mmu_fault(struct gk20a *g,
1100                 unsigned long engine_ids)
1101 {
1102         unsigned long end_jiffies = jiffies +
1103                 msecs_to_jiffies(gk20a_get_gr_idle_timeout(g));
1104         unsigned long delay = GR_IDLE_CHECK_DEFAULT;
1105         unsigned long engine_id;
1106         int ret;
1107
1108         /*
1109          * sched error prevents recovery, and ctxsw error will retrigger
1110          * every 100ms. Disable the sched error to allow recovery.
1111          */
1112         gk20a_writel(g, fifo_intr_en_0_r(),
1113                      0x7FFFFFFF & ~fifo_intr_en_0_sched_error_m());
1114         gk20a_writel(g, fifo_intr_0_r(),
1115                         fifo_intr_0_sched_error_reset_f());
1116
1117         /* trigger faults for all bad engines */
1118         for_each_set_bit(engine_id, &engine_ids, 32) {
1119                 if (engine_id > g->fifo.max_engines) {
1120                         WARN_ON(true);
1121                         break;
1122                 }
1123
1124                 gk20a_writel(g, fifo_trigger_mmu_fault_r(engine_id),
1125                              fifo_trigger_mmu_fault_id_f(
1126                              gk20a_engine_id_to_mmu_id(engine_id)) |
1127                              fifo_trigger_mmu_fault_enable_f(1));
1128         }
1129
1130         /* Wait for MMU fault to trigger */
1131         ret = -EBUSY;
1132         do {
1133                 if (gk20a_readl(g, fifo_intr_0_r()) &
1134                                 fifo_intr_0_mmu_fault_pending_f()) {
1135                         ret = 0;
1136                         break;
1137                 }
1138
1139                 usleep_range(delay, delay * 2);
1140                 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
1141         } while (time_before(jiffies, end_jiffies) ||
1142                         !tegra_platform_is_silicon());
1143
1144         if (ret)
1145                 gk20a_err(dev_from_gk20a(g), "mmu fault timeout");
1146
1147         /* release mmu fault trigger */
1148         for_each_set_bit(engine_id, &engine_ids, 32)
1149                 gk20a_writel(g, fifo_trigger_mmu_fault_r(engine_id), 0);
1150
1151         /* Re-enable sched error */
1152         gk20a_writel(g, fifo_intr_en_0_r(), 0x7FFFFFFF);
1153 }
1154
1155 u32 gk20a_fifo_engines_on_ch(struct gk20a *g, u32 hw_chid)
1156 {
1157         int i;
1158         u32 engines = 0;
1159
1160         for (i = 0; i < g->fifo.max_engines; i++) {
1161                 u32 status = gk20a_readl(g, fifo_engine_status_r(i));
1162                 u32 ctx_status =
1163                         fifo_engine_status_ctx_status_v(status);
1164                 bool type_ch = fifo_pbdma_status_id_type_v(status) ==
1165                         fifo_pbdma_status_id_type_chid_v();
1166                 bool busy = fifo_engine_status_engine_v(status) ==
1167                         fifo_engine_status_engine_busy_v();
1168                 u32 id = (ctx_status ==
1169                         fifo_engine_status_ctx_status_ctxsw_load_v()) ?
1170                         fifo_engine_status_next_id_v(status) :
1171                         fifo_engine_status_id_v(status);
1172
1173                 if (type_ch && busy && id == hw_chid)
1174                         engines |= BIT(i);
1175         }
1176
1177         return engines;
1178 }
1179
1180 void gk20a_fifo_recover_ch(struct gk20a *g, u32 hw_chid, bool verbose)
1181 {
1182         u32 engines = gk20a_fifo_engines_on_ch(g, hw_chid);
1183         if (engines)
1184                 gk20a_fifo_recover(g, engines, verbose);
1185         else {
1186                 int i;
1187                 struct channel_gk20a *ch =
1188                         g->fifo.channel + hw_chid;
1189
1190                 gk20a_disable_channel_no_update(ch);
1191                 for (i = 0; i < g->fifo.max_runlists; i++)
1192                         gk20a_fifo_update_runlist(g, i,
1193                                         hw_chid, false, false);
1194
1195                 if (gk20a_fifo_set_ctx_mmu_error(g, ch))
1196                         gk20a_debug_dump(g->dev);
1197         }
1198 }
1199
1200 void gk20a_fifo_recover(struct gk20a *g, u32 __engine_ids,
1201                 bool verbose)
1202 {
1203         unsigned long engine_id, i;
1204         unsigned long _engine_ids = __engine_ids;
1205         unsigned long engine_ids = 0;
1206
1207         if (verbose)
1208                 gk20a_debug_dump(g->dev);
1209
1210         /* store faulted engines in advance */
1211         g->fifo.mmu_fault_engines = 0;
1212         for_each_set_bit(engine_id, &_engine_ids, 32) {
1213                 bool ref_type_ch;
1214                 int ref_chid;
1215                 gk20a_fifo_get_faulty_channel(g, engine_id, &ref_chid,
1216                                               &ref_type_ch);
1217
1218                 /* Reset *all* engines that use the
1219                  * same channel as faulty engine */
1220                 for (i = 0; i < g->fifo.max_engines; i++) {
1221                         bool type_ch;
1222                         u32 chid;
1223                         gk20a_fifo_get_faulty_channel(g, i, &chid, &type_ch);
1224                         if (ref_type_ch == type_ch && ref_chid == chid) {
1225                                 engine_ids |= BIT(i);
1226                                 g->fifo.mmu_fault_engines |=
1227                                         BIT(gk20a_engine_id_to_mmu_id(i));
1228                         }
1229                 }
1230
1231         }
1232
1233         g->ops.fifo.trigger_mmu_fault(g, engine_ids);
1234 }
1235
1236
1237 static bool gk20a_fifo_handle_sched_error(struct gk20a *g)
1238 {
1239         u32 sched_error;
1240         u32 engine_id;
1241         int id = -1;
1242         bool non_chid = false;
1243
1244         /* read and reset the scheduler error register */
1245         sched_error = gk20a_readl(g, fifo_intr_sched_error_r());
1246         gk20a_writel(g, fifo_intr_0_r(), fifo_intr_0_sched_error_reset_f());
1247
1248         for (engine_id = 0; engine_id < g->fifo.max_engines; engine_id++) {
1249                 u32 status = gk20a_readl(g, fifo_engine_status_r(engine_id));
1250                 u32 ctx_status = fifo_engine_status_ctx_status_v(status);
1251                 bool failing_engine;
1252
1253                 /* we are interested in busy engines */
1254                 failing_engine = fifo_engine_status_engine_v(status) ==
1255                         fifo_engine_status_engine_busy_v();
1256
1257                 /* ..that are doing context switch */
1258                 failing_engine = failing_engine &&
1259                         (ctx_status ==
1260                                 fifo_engine_status_ctx_status_ctxsw_switch_v()
1261                         || ctx_status ==
1262                                 fifo_engine_status_ctx_status_ctxsw_save_v()
1263                         || ctx_status ==
1264                                 fifo_engine_status_ctx_status_ctxsw_load_v());
1265
1266                 if (failing_engine) {
1267                         id = (ctx_status ==
1268                                 fifo_engine_status_ctx_status_ctxsw_load_v()) ?
1269                                 fifo_engine_status_next_id_v(status) :
1270                                 fifo_engine_status_id_v(status);
1271                         non_chid = fifo_pbdma_status_id_type_v(status) !=
1272                                 fifo_pbdma_status_id_type_chid_v();
1273                         break;
1274                 }
1275         }
1276
1277         /* could not find the engine - should never happen */
1278         if (unlikely(engine_id >= g->fifo.max_engines))
1279                 goto err;
1280
1281         if (fifo_intr_sched_error_code_f(sched_error) ==
1282                         fifo_intr_sched_error_code_ctxsw_timeout_v()) {
1283                 struct fifo_gk20a *f = &g->fifo;
1284                 struct channel_gk20a *ch = &f->channel[id];
1285
1286                 if (non_chid) {
1287                         gk20a_fifo_recover(g, BIT(engine_id), true);
1288                         goto err;
1289                 }
1290
1291                 if (gk20a_channel_update_and_check_timeout(ch,
1292                         GRFIFO_TIMEOUT_CHECK_PERIOD_US / 1000)) {
1293                         gk20a_set_error_notifier(ch,
1294                                 NVHOST_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT);
1295                         gk20a_err(dev_from_gk20a(g),
1296                                 "fifo sched ctxsw timeout error:"
1297                                 "engine = %u, ch = %d", engine_id, id);
1298                         gk20a_fifo_recover(g, BIT(engine_id),
1299                                 ch->timeout_debug_dump);
1300                 } else {
1301                         gk20a_warn(dev_from_gk20a(g),
1302                                 "fifo is waiting for ctx switch for %d ms,"
1303                                 "ch = %d\n",
1304                                 ch->timeout_accumulated_ms,
1305                                 id);
1306                 }
1307                 return ch->timeout_debug_dump;
1308         }
1309 err:
1310         gk20a_err(dev_from_gk20a(g), "fifo sched error : 0x%08x, engine=%u, %s=%d",
1311                    sched_error, engine_id, non_chid ? "non-ch" : "ch", id);
1312
1313         return true;
1314 }
1315
1316 static u32 fifo_error_isr(struct gk20a *g, u32 fifo_intr)
1317 {
1318         bool print_channel_reset_log = false, reset_engine = false;
1319         struct device *dev = dev_from_gk20a(g);
1320         u32 handled = 0;
1321
1322         gk20a_dbg_fn("");
1323
1324         if (fifo_intr & fifo_intr_0_pio_error_pending_f()) {
1325                 /* pio mode is unused.  this shouldn't happen, ever. */
1326                 /* should we clear it or just leave it pending? */
1327                 gk20a_err(dev, "fifo pio error!\n");
1328                 BUG_ON(1);
1329         }
1330
1331         if (fifo_intr & fifo_intr_0_bind_error_pending_f()) {
1332                 u32 bind_error = gk20a_readl(g, fifo_intr_bind_error_r());
1333                 gk20a_err(dev, "fifo bind error: 0x%08x", bind_error);
1334                 print_channel_reset_log = true;
1335                 handled |= fifo_intr_0_bind_error_pending_f();
1336         }
1337
1338         if (fifo_intr & fifo_intr_0_sched_error_pending_f()) {
1339                 print_channel_reset_log = gk20a_fifo_handle_sched_error(g);
1340                 handled |= fifo_intr_0_sched_error_pending_f();
1341         }
1342
1343         if (fifo_intr & fifo_intr_0_chsw_error_pending_f()) {
1344                 gk20a_fifo_handle_chsw_fault(g);
1345                 handled |= fifo_intr_0_chsw_error_pending_f();
1346         }
1347
1348         if (fifo_intr & fifo_intr_0_mmu_fault_pending_f()) {
1349                 print_channel_reset_log = gk20a_fifo_handle_mmu_fault(g);
1350                 reset_engine  = true;
1351                 handled |= fifo_intr_0_mmu_fault_pending_f();
1352         }
1353
1354         if (fifo_intr & fifo_intr_0_dropped_mmu_fault_pending_f()) {
1355                 gk20a_fifo_handle_dropped_mmu_fault(g);
1356                 handled |= fifo_intr_0_dropped_mmu_fault_pending_f();
1357         }
1358
1359         print_channel_reset_log = !g->fifo.deferred_reset_pending
1360                         && print_channel_reset_log;
1361
1362         if (print_channel_reset_log) {
1363                 int engine_id;
1364                 gk20a_err(dev_from_gk20a(g),
1365                            "channel reset initated from %s", __func__);
1366                 for (engine_id = 0;
1367                      engine_id < g->fifo.max_engines;
1368                      engine_id++) {
1369                         gk20a_dbg_fn("enum:%d -> engine_id:%d", engine_id,
1370                                 g->fifo.engine_info[engine_id].engine_id);
1371                         fifo_pbdma_exception_status(g,
1372                                         &g->fifo.engine_info[engine_id]);
1373                         fifo_engine_exception_status(g,
1374                                         &g->fifo.engine_info[engine_id]);
1375                 }
1376         }
1377
1378         return handled;
1379 }
1380
1381 static u32 gk20a_fifo_handle_pbdma_intr(struct device *dev,
1382                                         struct gk20a *g,
1383                                         struct fifo_gk20a *f,
1384                                         u32 pbdma_id)
1385 {
1386         u32 pbdma_intr_0 = gk20a_readl(g, pbdma_intr_0_r(pbdma_id));
1387         u32 pbdma_intr_1 = gk20a_readl(g, pbdma_intr_1_r(pbdma_id));
1388         u32 handled = 0;
1389         bool reset = false;
1390
1391         gk20a_dbg_fn("");
1392
1393         gk20a_dbg(gpu_dbg_intr, "pbdma id intr pending %d %08x %08x", pbdma_id,
1394                         pbdma_intr_0, pbdma_intr_1);
1395         if (pbdma_intr_0) {
1396                 if ((f->intr.pbdma.device_fatal_0 |
1397                      f->intr.pbdma.channel_fatal_0 |
1398                      f->intr.pbdma.restartable_0) & pbdma_intr_0) {
1399                         gk20a_err(dev_from_gk20a(g),
1400                                 "pbdma_intr_0(%d):0x%08x PBH: %08x SHADOW: %08x M0: %08x",
1401                                 pbdma_id, pbdma_intr_0,
1402                                 gk20a_readl(g, pbdma_pb_header_r(pbdma_id)),
1403                                 gk20a_readl(g, pbdma_hdr_shadow_r(pbdma_id)),
1404                                 gk20a_readl(g, pbdma_method0_r(pbdma_id)));
1405                         reset = true;
1406                         handled |= ((f->intr.pbdma.device_fatal_0 |
1407                                      f->intr.pbdma.channel_fatal_0 |
1408                                      f->intr.pbdma.restartable_0) &
1409                                     pbdma_intr_0);
1410                 }
1411
1412                 gk20a_writel(g, pbdma_intr_0_r(pbdma_id), pbdma_intr_0);
1413         }
1414
1415         /* all intrs in _intr_1 are "host copy engine" related,
1416          * which gk20a doesn't have. for now just make them channel fatal. */
1417         if (pbdma_intr_1) {
1418                 dev_err(dev, "channel hce error: pbdma_intr_1(%d): 0x%08x",
1419                         pbdma_id, pbdma_intr_1);
1420                 reset = true;
1421                 gk20a_writel(g, pbdma_intr_1_r(pbdma_id), pbdma_intr_1);
1422         }
1423
1424         if (reset) {
1425                 /* Remove the channel from runlist */
1426                 u32 status = gk20a_readl(g, fifo_pbdma_status_r(pbdma_id));
1427                 u32 hw_chid = fifo_pbdma_status_id_v(status);
1428                 if (fifo_pbdma_status_id_type_v(status)
1429                                 == fifo_pbdma_status_id_type_chid_v()) {
1430                         gk20a_fifo_recover_ch(g, hw_chid, true);
1431                 }
1432         }
1433
1434         return handled;
1435 }
1436
1437 static u32 fifo_channel_isr(struct gk20a *g, u32 fifo_intr)
1438 {
1439         gk20a_channel_semaphore_wakeup(g);
1440         return fifo_intr_0_channel_intr_pending_f();
1441 }
1442
1443
1444 static u32 fifo_pbdma_isr(struct gk20a *g, u32 fifo_intr)
1445 {
1446         struct device *dev = dev_from_gk20a(g);
1447         struct fifo_gk20a *f = &g->fifo;
1448         u32 clear_intr = 0, i;
1449         u32 pbdma_pending = gk20a_readl(g, fifo_intr_pbdma_id_r());
1450
1451         for (i = 0; i < fifo_intr_pbdma_id_status__size_1_v(); i++) {
1452                 if (fifo_intr_pbdma_id_status_f(pbdma_pending, i)) {
1453                         gk20a_dbg(gpu_dbg_intr, "pbdma id %d intr pending", i);
1454                         clear_intr |=
1455                                 gk20a_fifo_handle_pbdma_intr(dev, g, f, i);
1456                 }
1457         }
1458         return fifo_intr_0_pbdma_intr_pending_f();
1459 }
1460
1461 void gk20a_fifo_isr(struct gk20a *g)
1462 {
1463         u32 error_intr_mask =
1464                 fifo_intr_0_bind_error_pending_f() |
1465                 fifo_intr_0_sched_error_pending_f() |
1466                 fifo_intr_0_chsw_error_pending_f() |
1467                 fifo_intr_0_fb_flush_timeout_pending_f() |
1468                 fifo_intr_0_dropped_mmu_fault_pending_f() |
1469                 fifo_intr_0_mmu_fault_pending_f() |
1470                 fifo_intr_0_lb_error_pending_f() |
1471                 fifo_intr_0_pio_error_pending_f();
1472
1473         u32 fifo_intr = gk20a_readl(g, fifo_intr_0_r());
1474         u32 clear_intr = 0;
1475
1476         /* note we're not actually in an "isr", but rather
1477          * in a threaded interrupt context... */
1478         mutex_lock(&g->fifo.intr.isr.mutex);
1479
1480         gk20a_dbg(gpu_dbg_intr, "fifo isr %08x\n", fifo_intr);
1481
1482         /* handle runlist update */
1483         if (fifo_intr & fifo_intr_0_runlist_event_pending_f()) {
1484                 gk20a_fifo_handle_runlist_event(g);
1485                 clear_intr |= fifo_intr_0_runlist_event_pending_f();
1486         }
1487         if (fifo_intr & fifo_intr_0_pbdma_intr_pending_f())
1488                 clear_intr |= fifo_pbdma_isr(g, fifo_intr);
1489
1490         if (unlikely(fifo_intr & error_intr_mask))
1491                 clear_intr = fifo_error_isr(g, fifo_intr);
1492
1493         gk20a_writel(g, fifo_intr_0_r(), clear_intr);
1494
1495         mutex_unlock(&g->fifo.intr.isr.mutex);
1496
1497         return;
1498 }
1499
1500 void gk20a_fifo_nonstall_isr(struct gk20a *g)
1501 {
1502         u32 fifo_intr = gk20a_readl(g, fifo_intr_0_r());
1503         u32 clear_intr = 0;
1504
1505         gk20a_dbg(gpu_dbg_intr, "fifo nonstall isr %08x\n", fifo_intr);
1506
1507         if (fifo_intr & fifo_intr_0_channel_intr_pending_f())
1508                 clear_intr |= fifo_channel_isr(g, fifo_intr);
1509
1510         gk20a_writel(g, fifo_intr_0_r(), clear_intr);
1511
1512         return;
1513 }
1514
1515 int gk20a_fifo_preempt_channel(struct gk20a *g, u32 hw_chid)
1516 {
1517         struct fifo_gk20a *f = &g->fifo;
1518         unsigned long end_jiffies = jiffies
1519                 + msecs_to_jiffies(gk20a_get_gr_idle_timeout(g));
1520         u32 delay = GR_IDLE_CHECK_DEFAULT;
1521         u32 ret = 0;
1522         u32 token = PMU_INVALID_MUTEX_OWNER_ID;
1523         u32 mutex_ret = 0;
1524         u32 i;
1525
1526         gk20a_dbg_fn("%d", hw_chid);
1527
1528         /* we have no idea which runlist we are using. lock all */
1529         for (i = 0; i < g->fifo.max_runlists; i++)
1530                 mutex_lock(&f->runlist_info[i].mutex);
1531
1532         mutex_ret = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1533
1534         /* issue preempt */
1535         gk20a_writel(g, fifo_preempt_r(),
1536                 fifo_preempt_chid_f(hw_chid) |
1537                 fifo_preempt_type_channel_f());
1538
1539         /* wait for preempt */
1540         ret = -EBUSY;
1541         do {
1542                 if (!(gk20a_readl(g, fifo_preempt_r()) &
1543                         fifo_preempt_pending_true_f())) {
1544                         ret = 0;
1545                         break;
1546                 }
1547
1548                 usleep_range(delay, delay * 2);
1549                 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
1550         } while (time_before(jiffies, end_jiffies) ||
1551                         !tegra_platform_is_silicon());
1552
1553         if (ret) {
1554                 struct channel_gk20a *ch = &g->fifo.channel[hw_chid];
1555
1556                 gk20a_err(dev_from_gk20a(g), "preempt channel %d timeout\n",
1557                             hw_chid);
1558
1559                 gk20a_set_error_notifier(ch,
1560                                 NVHOST_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT);
1561                 gk20a_fifo_recover_ch(g, hw_chid, true);
1562         }
1563
1564         if (!mutex_ret)
1565                 pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1566
1567         for (i = 0; i < g->fifo.max_runlists; i++)
1568                 mutex_unlock(&f->runlist_info[i].mutex);
1569
1570         return ret;
1571 }
1572
1573 int gk20a_fifo_enable_engine_activity(struct gk20a *g,
1574                                 struct fifo_engine_info_gk20a *eng_info)
1575 {
1576         u32 token = PMU_INVALID_MUTEX_OWNER_ID;
1577         u32 mutex_ret;
1578         u32 enable;
1579
1580         gk20a_dbg_fn("");
1581
1582         mutex_ret = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1583
1584         enable = gk20a_readl(g, fifo_sched_disable_r());
1585         enable &= ~(fifo_sched_disable_true_v() >> eng_info->runlist_id);
1586         gk20a_writel(g, fifo_sched_disable_r(), enable);
1587
1588         if (!mutex_ret)
1589                 pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1590
1591         gk20a_dbg_fn("done");
1592         return 0;
1593 }
1594
1595 int gk20a_fifo_disable_engine_activity(struct gk20a *g,
1596                                 struct fifo_engine_info_gk20a *eng_info,
1597                                 bool wait_for_idle)
1598 {
1599         u32 gr_stat, pbdma_stat, chan_stat, eng_stat, ctx_stat;
1600         u32 pbdma_chid = ~0, engine_chid = ~0, disable;
1601         u32 token = PMU_INVALID_MUTEX_OWNER_ID;
1602         u32 mutex_ret;
1603         u32 err = 0;
1604
1605         gk20a_dbg_fn("");
1606
1607         gr_stat =
1608                 gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id));
1609         if (fifo_engine_status_engine_v(gr_stat) ==
1610             fifo_engine_status_engine_busy_v() && !wait_for_idle)
1611                 return -EBUSY;
1612
1613         mutex_ret = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1614
1615         disable = gk20a_readl(g, fifo_sched_disable_r());
1616         disable = set_field(disable,
1617                         fifo_sched_disable_runlist_m(eng_info->runlist_id),
1618                         fifo_sched_disable_runlist_f(fifo_sched_disable_true_v(),
1619                                 eng_info->runlist_id));
1620         gk20a_writel(g, fifo_sched_disable_r(), disable);
1621
1622         /* chid from pbdma status */
1623         pbdma_stat = gk20a_readl(g, fifo_pbdma_status_r(eng_info->pbdma_id));
1624         chan_stat  = fifo_pbdma_status_chan_status_v(pbdma_stat);
1625         if (chan_stat == fifo_pbdma_status_chan_status_valid_v() ||
1626             chan_stat == fifo_pbdma_status_chan_status_chsw_save_v())
1627                 pbdma_chid = fifo_pbdma_status_id_v(pbdma_stat);
1628         else if (chan_stat == fifo_pbdma_status_chan_status_chsw_load_v() ||
1629                  chan_stat == fifo_pbdma_status_chan_status_chsw_switch_v())
1630                 pbdma_chid = fifo_pbdma_status_next_id_v(pbdma_stat);
1631
1632         if (pbdma_chid != ~0) {
1633                 err = gk20a_fifo_preempt_channel(g, pbdma_chid);
1634                 if (err)
1635                         goto clean_up;
1636         }
1637
1638         /* chid from engine status */
1639         eng_stat = gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id));
1640         ctx_stat  = fifo_engine_status_ctx_status_v(eng_stat);
1641         if (ctx_stat == fifo_engine_status_ctx_status_valid_v() ||
1642             ctx_stat == fifo_engine_status_ctx_status_ctxsw_save_v())
1643                 engine_chid = fifo_engine_status_id_v(eng_stat);
1644         else if (ctx_stat == fifo_engine_status_ctx_status_ctxsw_load_v() ||
1645                  ctx_stat == fifo_engine_status_ctx_status_ctxsw_switch_v())
1646                 engine_chid = fifo_engine_status_next_id_v(eng_stat);
1647
1648         if (engine_chid != ~0 && engine_chid != pbdma_chid) {
1649                 err = gk20a_fifo_preempt_channel(g, engine_chid);
1650                 if (err)
1651                         goto clean_up;
1652         }
1653
1654 clean_up:
1655         if (!mutex_ret)
1656                 pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1657
1658         if (err) {
1659                 gk20a_dbg_fn("failed");
1660                 if (gk20a_fifo_enable_engine_activity(g, eng_info))
1661                         gk20a_err(dev_from_gk20a(g),
1662                                 "failed to enable gr engine activity\n");
1663         } else {
1664                 gk20a_dbg_fn("done");
1665         }
1666         return err;
1667 }
1668
1669 static void gk20a_fifo_runlist_reset_engines(struct gk20a *g, u32 runlist_id)
1670 {
1671         struct fifo_gk20a *f = &g->fifo;
1672         u32 engines = 0;
1673         int i;
1674
1675         for (i = 0; i < f->max_engines; i++) {
1676                 u32 status = gk20a_readl(g, fifo_engine_status_r(i));
1677                 bool engine_busy = fifo_engine_status_engine_v(status) ==
1678                         fifo_engine_status_engine_busy_v();
1679
1680                 if (engine_busy &&
1681                     (f->engine_info[i].runlist_id == runlist_id))
1682                         engines |= BIT(i);
1683         }
1684         gk20a_fifo_recover(g, engines, true);
1685 }
1686
1687 static int gk20a_fifo_runlist_wait_pending(struct gk20a *g, u32 runlist_id)
1688 {
1689         struct fifo_runlist_info_gk20a *runlist;
1690         u32 remain;
1691         bool pending;
1692
1693         runlist = &g->fifo.runlist_info[runlist_id];
1694         remain = wait_event_timeout(runlist->runlist_wq,
1695                 ((pending = gk20a_readl(g, fifo_eng_runlist_r(runlist_id)) &
1696                         fifo_eng_runlist_pending_true_f()) == 0),
1697                 msecs_to_jiffies(gk20a_get_gr_idle_timeout(g)));
1698
1699         if (remain == 0 && pending != 0)
1700                 return -ETIMEDOUT;
1701
1702         return 0;
1703 }
1704
1705 static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
1706                                             u32 hw_chid, bool add,
1707                                             bool wait_for_finish)
1708 {
1709         u32 ret = 0;
1710         struct device *d = dev_from_gk20a(g);
1711         struct fifo_gk20a *f = &g->fifo;
1712         struct fifo_runlist_info_gk20a *runlist = NULL;
1713         u32 *runlist_entry_base = NULL;
1714         u32 *runlist_entry = NULL;
1715         phys_addr_t runlist_pa;
1716         u32 old_buf, new_buf;
1717         u32 chid;
1718         u32 count = 0;
1719         runlist = &f->runlist_info[runlist_id];
1720
1721         /* valid channel, add/remove it from active list.
1722            Otherwise, keep active list untouched for suspend/resume. */
1723         if (hw_chid != ~0) {
1724                 if (add) {
1725                         if (test_and_set_bit(hw_chid,
1726                                 runlist->active_channels) == 1)
1727                                 return 0;
1728                 } else {
1729                         if (test_and_clear_bit(hw_chid,
1730                                 runlist->active_channels) == 0)
1731                                 return 0;
1732                 }
1733         }
1734
1735         old_buf = runlist->cur_buffer;
1736         new_buf = !runlist->cur_buffer;
1737
1738         gk20a_dbg_info("runlist_id : %d, switch to new buffer 0x%16llx",
1739                 runlist_id, runlist->mem[new_buf].iova);
1740
1741         runlist_pa = gk20a_get_phys_from_iova(d, runlist->mem[new_buf].iova);
1742         if (!runlist_pa) {
1743                 ret = -EINVAL;
1744                 goto clean_up;
1745         }
1746
1747         runlist_entry_base = runlist->mem[new_buf].cpuva;
1748         if (!runlist_entry_base) {
1749                 ret = -ENOMEM;
1750                 goto clean_up;
1751         }
1752
1753         if (hw_chid != ~0 || /* add/remove a valid channel */
1754             add /* resume to add all channels back */) {
1755                 runlist_entry = runlist_entry_base;
1756                 for_each_set_bit(chid,
1757                         runlist->active_channels, f->num_channels) {
1758                         gk20a_dbg_info("add channel %d to runlist", chid);
1759                         runlist_entry[0] = chid;
1760                         runlist_entry[1] = 0;
1761                         runlist_entry += 2;
1762                         count++;
1763                 }
1764         } else  /* suspend to remove all channels */
1765                 count = 0;
1766
1767         if (count != 0) {
1768                 gk20a_writel(g, fifo_runlist_base_r(),
1769                         fifo_runlist_base_ptr_f(u64_lo32(runlist_pa >> 12)) |
1770                         fifo_runlist_base_target_vid_mem_f());
1771         }
1772
1773         gk20a_writel(g, fifo_runlist_r(),
1774                 fifo_runlist_engine_f(runlist_id) |
1775                 fifo_eng_runlist_length_f(count));
1776
1777         if (wait_for_finish) {
1778                 ret = gk20a_fifo_runlist_wait_pending(g, runlist_id);
1779
1780                 if (ret == -ETIMEDOUT) {
1781                         gk20a_err(dev_from_gk20a(g),
1782                                    "runlist update timeout");
1783
1784                         gk20a_fifo_runlist_reset_engines(g, runlist_id);
1785
1786                         /* engine reset needs the lock. drop it */
1787                         mutex_unlock(&runlist->mutex);
1788                         /* wait until the runlist is active again */
1789                         ret = gk20a_fifo_runlist_wait_pending(g, runlist_id);
1790                         /* get the lock back. at this point everything should
1791                          * should be fine */
1792                         mutex_lock(&runlist->mutex);
1793
1794                         if (ret)
1795                                 gk20a_err(dev_from_gk20a(g),
1796                                            "runlist update failed: %d", ret);
1797                 } else if (ret == -EINTR)
1798                         gk20a_err(dev_from_gk20a(g),
1799                                    "runlist update interrupted");
1800         }
1801
1802         runlist->cur_buffer = new_buf;
1803
1804 clean_up:
1805         return ret;
1806 }
1807
1808 /* add/remove a channel from runlist
1809    special cases below: runlist->active_channels will NOT be changed.
1810    (hw_chid == ~0 && !add) means remove all active channels from runlist.
1811    (hw_chid == ~0 &&  add) means restore all active channels on runlist. */
1812 int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 hw_chid,
1813                               bool add, bool wait_for_finish)
1814 {
1815         struct fifo_runlist_info_gk20a *runlist = NULL;
1816         struct fifo_gk20a *f = &g->fifo;
1817         u32 token = PMU_INVALID_MUTEX_OWNER_ID;
1818         u32 mutex_ret;
1819         u32 ret = 0;
1820
1821         runlist = &f->runlist_info[runlist_id];
1822
1823         mutex_lock(&runlist->mutex);
1824
1825         mutex_ret = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1826
1827         ret = gk20a_fifo_update_runlist_locked(g, runlist_id, hw_chid, add,
1828                                                wait_for_finish);
1829
1830         if (!mutex_ret)
1831                 pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1832
1833         mutex_unlock(&runlist->mutex);
1834         return ret;
1835 }
1836
1837 int gk20a_fifo_suspend(struct gk20a *g)
1838 {
1839         gk20a_dbg_fn("");
1840
1841         /* stop bar1 snooping */
1842         gk20a_writel(g, fifo_bar1_base_r(),
1843                         fifo_bar1_base_valid_false_f());
1844
1845         /* disable fifo intr */
1846         gk20a_writel(g, fifo_intr_en_0_r(), 0);
1847         gk20a_writel(g, fifo_intr_en_1_r(), 0);
1848
1849         gk20a_dbg_fn("done");
1850         return 0;
1851 }
1852
1853 bool gk20a_fifo_mmu_fault_pending(struct gk20a *g)
1854 {
1855         if (gk20a_readl(g, fifo_intr_0_r()) &
1856                         fifo_intr_0_mmu_fault_pending_f())
1857                 return true;
1858         else
1859                 return false;
1860 }
1861
1862 int gk20a_fifo_wait_engine_idle(struct gk20a *g)
1863 {
1864         unsigned long end_jiffies = jiffies +
1865                 msecs_to_jiffies(gk20a_get_gr_idle_timeout(g));
1866         unsigned long delay = GR_IDLE_CHECK_DEFAULT;
1867         int ret = -ETIMEDOUT;
1868         u32 i;
1869         struct device *d = dev_from_gk20a(g);
1870
1871         gk20a_dbg_fn("");
1872
1873         for (i = 0; i < fifo_engine_status__size_1_v(); i++) {
1874                 do {
1875                         u32 status = gk20a_readl(g, fifo_engine_status_r(i));
1876                         if (!fifo_engine_status_engine_v(status)) {
1877                                 ret = 0;
1878                                 break;
1879                         }
1880
1881                         usleep_range(delay, delay * 2);
1882                         delay = min_t(unsigned long,
1883                                         delay << 1, GR_IDLE_CHECK_MAX);
1884                 } while (time_before(jiffies, end_jiffies) ||
1885                                 !tegra_platform_is_silicon());
1886                 if (ret) {
1887                         gk20a_err(d, "cannot idle engine %u\n", i);
1888                         break;
1889                 }
1890         }
1891
1892         gk20a_dbg_fn("done");
1893
1894         return ret;
1895 }
1896
1897 void gk20a_init_fifo(struct gpu_ops *gops)
1898 {
1899         gk20a_init_channel(gops);
1900         gops->fifo.trigger_mmu_fault = gk20a_fifo_trigger_mmu_fault;
1901 }