drivers/gpu/nvgpu/gk20a/fifo_gk20a.c

   1 /*
   2  * GK20A Graphics FIFO (gr host)
   3  *
   4  * Copyright (c) 2011-2015, NVIDIA CORPORATION.  All rights reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along with
  16  * this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18  */
  19 #include <linux/delay.h>
  20 #include <linux/slab.h>
  21 #include <linux/scatterlist.h>
  22 #include <trace/events/gk20a.h>
  23 #include <linux/dma-mapping.h>
  24 #include <linux/nvhost.h>
  25
  26 #include "gk20a.h"
  27 #include "debug_gk20a.h"
  28 #include "semaphore_gk20a.h"
  29 #include "hw_fifo_gk20a.h"
  30 #include "hw_pbdma_gk20a.h"
  31 #include "hw_ccsr_gk20a.h"
  32 #include "hw_ram_gk20a.h"
  33 #include "hw_proj_gk20a.h"
  34 #include "hw_top_gk20a.h"
  35 #include "hw_mc_gk20a.h"
  36 #include "hw_gr_gk20a.h"
  37 #define FECS_METHOD_WFI_RESTORE 0x80000
  38
  39 static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
  40                                             u32 hw_chid, bool add,
  41                                             bool wait_for_finish);
  42
  43 /*
  44  * Link engine IDs to MMU IDs and vice versa.
  45  */
  46
  47 static inline u32 gk20a_engine_id_to_mmu_id(u32 engine_id)
  48 {
  49         switch (engine_id) {
  50         case ENGINE_GR_GK20A:
  51                 return 0x00;
  52         case ENGINE_CE2_GK20A:
  53                 return 0x1b;
  54         default:
  55                 return ~0;
  56         }
  57 }
  58
  59 static inline u32 gk20a_mmu_id_to_engine_id(u32 engine_id)
  60 {
  61         switch (engine_id) {
  62         case 0x00:
  63                 return ENGINE_GR_GK20A;
  64         case 0x1b:
  65                 return ENGINE_CE2_GK20A;
  66         default:
  67                 return ~0;
  68         }
  69 }
  70
  71
  72 static int init_engine_info(struct fifo_gk20a *f)
  73 {
  74         struct gk20a *g = f->g;
  75         struct device *d = dev_from_gk20a(g);
  76         u32 i;
  77         u32 max_info_entries = top_device_info__size_1_v();
  78
  79         gk20a_dbg_fn("");
  80
  81         /* all we really care about finding is the graphics entry    */
  82         /* especially early on in sim it probably thinks it has more */
  83         f->num_engines = 2;
  84
  85         for (i = 0; i < max_info_entries; i++) {
  86                 struct fifo_engine_info_gk20a *info = NULL;
  87                 u32 table_entry = gk20a_readl(f->g, top_device_info_r(i));
  88                 u32 entry = top_device_info_entry_v(table_entry);
  89                 u32 engine_enum;
  90                 int pbdma_id;
  91                 u32 runlist_bit;
  92
  93                 if (entry != top_device_info_entry_enum_v())
  94                         continue;
  95
  96                 /* we only care about GR engine here */
  97                 engine_enum = top_device_info_engine_enum_v(table_entry);
  98                 if (engine_enum >= ENGINE_INVAL_GK20A)
  99                         continue;
 100
 101                 gk20a_dbg_info("info: engine_id %d",
 102                                 top_device_info_engine_enum_v(table_entry));
 103                 info = &g->fifo.engine_info[engine_enum];
 104
 105                 info->runlist_id =
 106                         top_device_info_runlist_enum_v(table_entry);
 107                 gk20a_dbg_info("gr info: runlist_id %d", info->runlist_id);
 108
 109                 info->engine_id =
 110                         top_device_info_engine_enum_v(table_entry);
 111                 gk20a_dbg_info("gr info: engine_id %d", info->engine_id);
 112
 113                 runlist_bit = 1 << info->runlist_id;
 114
 115                 for (pbdma_id = 0; pbdma_id < f->num_pbdma; pbdma_id++) {
 116                         gk20a_dbg_info("gr info: pbdma_map[%d]=%d",
 117                                 pbdma_id, f->pbdma_map[pbdma_id]);
 118                         if (f->pbdma_map[pbdma_id] & runlist_bit)
 119                                 break;
 120                 }
 121
 122                 if (pbdma_id == f->num_pbdma) {
 123                         gk20a_err(d, "busted pbmda map");
 124                         return -EINVAL;
 125                 }
 126                 info->pbdma_id = pbdma_id;
 127
 128                 info->intr_id =
 129                         top_device_info_intr_enum_v(table_entry);
 130                 gk20a_dbg_info("gr info: intr_id %d", info->intr_id);
 131
 132                 info->reset_id =
 133                         top_device_info_reset_enum_v(table_entry);
 134                 gk20a_dbg_info("gr info: reset_id %d",
 135                                 info->reset_id);
 136
 137         }
 138
 139         return 0;
 140 }
 141
 142 u32 gk20a_fifo_engine_interrupt_mask(struct gk20a *g)
 143 {
 144         u32 eng_intr_mask = 0;
 145         int i = 0;
 146
 147         for (i = 0; i < g->fifo.max_engines; i++) {
 148                 u32 intr_id = g->fifo.engine_info[i].intr_id;
 149                 if (i == ENGINE_CE2_GK20A &&
 150                         (!g->ops.ce2.isr_stall || !g->ops.ce2.isr_nonstall))
 151                         continue;
 152
 153                 if (intr_id)
 154                         eng_intr_mask |= BIT(intr_id);
 155         }
 156
 157         return eng_intr_mask;
 158 }
 159
 160 static void gk20a_remove_fifo_support(struct fifo_gk20a *f)
 161 {
 162         struct gk20a *g = f->g;
 163         struct fifo_engine_info_gk20a *engine_info;
 164         struct fifo_runlist_info_gk20a *runlist;
 165         u32 runlist_id;
 166         u32 i;
 167
 168         gk20a_dbg_fn("");
 169
 170         if (f->channel) {
 171                 int c;
 172                 for (c = 0; c < f->num_channels; c++) {
 173                         if (f->channel[c].remove_support)
 174                                 f->channel[c].remove_support(f->channel+c);
 175                 }
 176                 kfree(f->channel);
 177         }
 178         gk20a_gmmu_unmap_free(&g->mm.bar1.vm, &f->userd);
 179
 180         engine_info = f->engine_info + ENGINE_GR_GK20A;
 181         runlist_id = engine_info->runlist_id;
 182         runlist = &f->runlist_info[runlist_id];
 183
 184         for (i = 0; i < MAX_RUNLIST_BUFFERS; i++)
 185                 gk20a_gmmu_free(g, &runlist->mem[i]);
 186
 187         kfree(runlist->active_channels);
 188         kfree(runlist->active_tsgs);
 189
 190         kfree(f->runlist_info);
 191         kfree(f->pbdma_map);
 192         kfree(f->engine_info);
 193 }
 194
 195 /* reads info from hardware and fills in pbmda exception info record */
 196 static inline void get_exception_pbdma_info(
 197         struct gk20a *g,
 198         struct fifo_engine_info_gk20a *eng_info)
 199 {
 200         struct fifo_pbdma_exception_info_gk20a *e =
 201                 &eng_info->pbdma_exception_info;
 202
 203         u32 pbdma_status_r = e->status_r = gk20a_readl(g,
 204                    fifo_pbdma_status_r(eng_info->pbdma_id));
 205         e->id = fifo_pbdma_status_id_v(pbdma_status_r); /* vs. id_hw_v()? */
 206         e->id_is_chid = fifo_pbdma_status_id_type_v(pbdma_status_r) ==
 207                 fifo_pbdma_status_id_type_chid_v();
 208         e->chan_status_v  = fifo_pbdma_status_chan_status_v(pbdma_status_r);
 209         e->next_id_is_chid =
 210                 fifo_pbdma_status_next_id_type_v(pbdma_status_r) ==
 211                 fifo_pbdma_status_next_id_type_chid_v();
 212         e->next_id = fifo_pbdma_status_next_id_v(pbdma_status_r);
 213         e->chsw_in_progress =
 214                 fifo_pbdma_status_chsw_v(pbdma_status_r) ==
 215                 fifo_pbdma_status_chsw_in_progress_v();
 216 }
 217
 218 static void fifo_pbdma_exception_status(struct gk20a *g,
 219         struct fifo_engine_info_gk20a *eng_info)
 220 {
 221         struct fifo_pbdma_exception_info_gk20a *e;
 222         get_exception_pbdma_info(g, eng_info);
 223         e = &eng_info->pbdma_exception_info;
 224
 225         gk20a_dbg_fn("pbdma_id %d, "
 226                       "id_type %s, id %d, chan_status %d, "
 227                       "next_id_type %s, next_id %d, "
 228                       "chsw_in_progress %d",
 229                       eng_info->pbdma_id,
 230                       e->id_is_chid ? "chid" : "tsgid", e->id, e->chan_status_v,
 231                       e->next_id_is_chid ? "chid" : "tsgid", e->next_id,
 232                       e->chsw_in_progress);
 233 }
 234
 235 /* reads info from hardware and fills in pbmda exception info record */
 236 static inline void get_exception_engine_info(
 237         struct gk20a *g,
 238         struct fifo_engine_info_gk20a *eng_info)
 239 {
 240         struct fifo_engine_exception_info_gk20a *e =
 241                 &eng_info->engine_exception_info;
 242         u32 engine_status_r = e->status_r =
 243                 gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id));
 244         e->id = fifo_engine_status_id_v(engine_status_r); /* vs. id_hw_v()? */
 245         e->id_is_chid = fifo_engine_status_id_type_v(engine_status_r) ==
 246                 fifo_engine_status_id_type_chid_v();
 247         e->ctx_status_v = fifo_engine_status_ctx_status_v(engine_status_r);
 248         e->faulted =
 249                 fifo_engine_status_faulted_v(engine_status_r) ==
 250                 fifo_engine_status_faulted_true_v();
 251         e->idle =
 252                 fifo_engine_status_engine_v(engine_status_r) ==
 253                 fifo_engine_status_engine_idle_v();
 254         e->ctxsw_in_progress =
 255                 fifo_engine_status_ctxsw_v(engine_status_r) ==
 256                 fifo_engine_status_ctxsw_in_progress_v();
 257 }
 258
 259 static void fifo_engine_exception_status(struct gk20a *g,
 260                                struct fifo_engine_info_gk20a *eng_info)
 261 {
 262         struct fifo_engine_exception_info_gk20a *e;
 263         get_exception_engine_info(g, eng_info);
 264         e = &eng_info->engine_exception_info;
 265
 266         gk20a_dbg_fn("engine_id %d, id_type %s, id %d, ctx_status %d, "
 267                       "faulted %d, idle %d, ctxsw_in_progress %d, ",
 268                       eng_info->engine_id, e->id_is_chid ? "chid" : "tsgid",
 269                       e->id, e->ctx_status_v,
 270                       e->faulted, e->idle,  e->ctxsw_in_progress);
 271 }
 272
 273 static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
 274 {
 275         struct fifo_engine_info_gk20a *engine_info;
 276         struct fifo_runlist_info_gk20a *runlist;
 277         struct device *d = dev_from_gk20a(g);
 278         u32 runlist_id;
 279         u32 i;
 280         u64 runlist_size;
 281
 282         gk20a_dbg_fn("");
 283
 284         f->max_runlists = fifo_eng_runlist_base__size_1_v();
 285         f->runlist_info = kzalloc(sizeof(struct fifo_runlist_info_gk20a) *
 286                                   f->max_runlists, GFP_KERNEL);
 287         if (!f->runlist_info)
 288                 goto clean_up;
 289
 290         engine_info = f->engine_info + ENGINE_GR_GK20A;
 291         runlist_id = engine_info->runlist_id;
 292         runlist = &f->runlist_info[runlist_id];
 293
 294         runlist->active_channels =
 295                 kzalloc(DIV_ROUND_UP(f->num_channels, BITS_PER_BYTE),
 296                         GFP_KERNEL);
 297         if (!runlist->active_channels)
 298                 goto clean_up_runlist_info;
 299
 300         runlist->active_tsgs =
 301                 kzalloc(DIV_ROUND_UP(f->num_channels, BITS_PER_BYTE),
 302                         GFP_KERNEL);
 303         if (!runlist->active_tsgs)
 304                 goto clean_up_runlist_info;
 305
 306         runlist_size  = ram_rl_entry_size_v() * f->num_channels;
 307         for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
 308                 int err = gk20a_gmmu_alloc(g, runlist_size, &runlist->mem[i]);
 309                 if (err) {
 310                         dev_err(d, "memory allocation failed\n");
 311                         goto clean_up_runlist;
 312                 }
 313         }
 314         mutex_init(&runlist->mutex);
 315
 316         /* None of buffers is pinned if this value doesn't change.
 317             Otherwise, one of them (cur_buffer) must have been pinned. */
 318         runlist->cur_buffer = MAX_RUNLIST_BUFFERS;
 319
 320         gk20a_dbg_fn("done");
 321         return 0;
 322
 323 clean_up_runlist:
 324         for (i = 0; i < MAX_RUNLIST_BUFFERS; i++)
 325                 gk20a_gmmu_free(g, &runlist->mem[i]);
 326
 327         kfree(runlist->active_channels);
 328         runlist->active_channels = NULL;
 329
 330 clean_up_runlist_info:
 331         kfree(f->runlist_info);
 332         f->runlist_info = NULL;
 333
 334 clean_up:
 335         gk20a_dbg_fn("fail");
 336         return -ENOMEM;
 337 }
 338
 339 #define GRFIFO_TIMEOUT_CHECK_PERIOD_US 100000
 340
 341 int gk20a_init_fifo_reset_enable_hw(struct gk20a *g)
 342 {
 343         u32 intr_stall;
 344         u32 mask;
 345         u32 timeout;
 346         int i;
 347
 348         gk20a_dbg_fn("");
 349         /* enable pmc pfifo */
 350         gk20a_reset(g, mc_enable_pfifo_enabled_f()
 351                         | mc_enable_ce2_enabled_f());
 352
 353         if (g->ops.clock_gating.slcg_ce2_load_gating_prod)
 354                 g->ops.clock_gating.slcg_ce2_load_gating_prod(g,
 355                                 g->slcg_enabled);
 356         if (g->ops.clock_gating.slcg_fifo_load_gating_prod)
 357                 g->ops.clock_gating.slcg_fifo_load_gating_prod(g,
 358                                 g->slcg_enabled);
 359         if (g->ops.clock_gating.blcg_fifo_load_gating_prod)
 360                 g->ops.clock_gating.blcg_fifo_load_gating_prod(g,
 361                                 g->blcg_enabled);
 362
 363         /* enable pbdma */
 364         mask = 0;
 365         for (i = 0; i < proj_host_num_pbdma_v(); ++i)
 366                 mask |= mc_enable_pb_sel_f(mc_enable_pb_0_enabled_v(), i);
 367         gk20a_writel(g, mc_enable_pb_r(), mask);
 368
 369         /* enable pfifo interrupt */
 370         gk20a_writel(g, fifo_intr_0_r(), 0xFFFFFFFF);
 371         gk20a_writel(g, fifo_intr_en_0_r(), 0x7FFFFFFF);
 372         gk20a_writel(g, fifo_intr_en_1_r(), 0x80000000);
 373
 374         /* enable pbdma interrupt */
 375         mask = 0;
 376         for (i = 0; i < proj_host_num_pbdma_v(); i++) {
 377                 intr_stall = gk20a_readl(g, pbdma_intr_stall_r(i));
 378                 intr_stall &= ~pbdma_intr_stall_lbreq_enabled_f();
 379                 gk20a_writel(g, pbdma_intr_stall_r(i), intr_stall);
 380                 gk20a_writel(g, pbdma_intr_0_r(i), 0xFFFFFFFF);
 381                 gk20a_writel(g, pbdma_intr_en_0_r(i),
 382                         ~pbdma_intr_en_0_lbreq_enabled_f());
 383                 gk20a_writel(g, pbdma_intr_1_r(i), 0xFFFFFFFF);
 384                 gk20a_writel(g, pbdma_intr_en_1_r(i),
 385                         ~pbdma_intr_en_0_lbreq_enabled_f());
 386         }
 387
 388         /* TBD: apply overrides */
 389
 390         /* TBD: BLCG prod */
 391
 392         /* reset runlist interrupts */
 393         gk20a_writel(g, fifo_intr_runlist_r(), ~0);
 394
 395         /* TBD: do we need those? */
 396         timeout = gk20a_readl(g, fifo_fb_timeout_r());
 397         timeout = set_field(timeout, fifo_fb_timeout_period_m(),
 398                         fifo_fb_timeout_period_max_f());
 399         gk20a_writel(g, fifo_fb_timeout_r(), timeout);
 400
 401         for (i = 0; i < pbdma_timeout__size_1_v(); i++) {
 402                 timeout = gk20a_readl(g, pbdma_timeout_r(i));
 403                 timeout = set_field(timeout, pbdma_timeout_period_m(),
 404                                     pbdma_timeout_period_max_f());
 405                 gk20a_writel(g, pbdma_timeout_r(i), timeout);
 406         }
 407
 408         if (g->ops.fifo.apply_pb_timeout)
 409                 g->ops.fifo.apply_pb_timeout(g);
 410
 411         timeout = GRFIFO_TIMEOUT_CHECK_PERIOD_US |
 412                         fifo_eng_timeout_detection_enabled_f();
 413         gk20a_writel(g, fifo_eng_timeout_r(), timeout);
 414
 415         gk20a_dbg_fn("done");
 416
 417         return 0;
 418 }
 419
 420 static void gk20a_init_fifo_pbdma_intr_descs(struct fifo_gk20a *f)
 421 {
 422         /* These are all errors which indicate something really wrong
 423          * going on in the device. */
 424         f->intr.pbdma.device_fatal_0 =
 425                 pbdma_intr_0_memreq_pending_f() |
 426                 pbdma_intr_0_memack_timeout_pending_f() |
 427                 pbdma_intr_0_memack_extra_pending_f() |
 428                 pbdma_intr_0_memdat_timeout_pending_f() |
 429                 pbdma_intr_0_memdat_extra_pending_f() |
 430                 pbdma_intr_0_memflush_pending_f() |
 431                 pbdma_intr_0_memop_pending_f() |
 432                 pbdma_intr_0_lbconnect_pending_f() |
 433                 pbdma_intr_0_lback_timeout_pending_f() |
 434                 pbdma_intr_0_lback_extra_pending_f() |
 435                 pbdma_intr_0_lbdat_timeout_pending_f() |
 436                 pbdma_intr_0_lbdat_extra_pending_f() |
 437                 pbdma_intr_0_xbarconnect_pending_f() |
 438                 pbdma_intr_0_pri_pending_f();
 439
 440         /* These are data parsing, framing errors or others which can be
 441          * recovered from with intervention... or just resetting the
 442          * channel. */
 443         f->intr.pbdma.channel_fatal_0 =
 444                 pbdma_intr_0_gpfifo_pending_f() |
 445                 pbdma_intr_0_gpptr_pending_f() |
 446                 pbdma_intr_0_gpentry_pending_f() |
 447                 pbdma_intr_0_gpcrc_pending_f() |
 448                 pbdma_intr_0_pbptr_pending_f() |
 449                 pbdma_intr_0_pbentry_pending_f() |
 450                 pbdma_intr_0_pbcrc_pending_f() |
 451                 pbdma_intr_0_method_pending_f() |
 452                 pbdma_intr_0_methodcrc_pending_f() |
 453                 pbdma_intr_0_pbseg_pending_f() |
 454                 pbdma_intr_0_signature_pending_f();
 455
 456         /* Can be used for sw-methods, or represents
 457          * a recoverable timeout. */
 458         f->intr.pbdma.restartable_0 =
 459                 pbdma_intr_0_device_pending_f() |
 460                 pbdma_intr_0_acquire_pending_f();
 461 }
 462
 463 static int gk20a_init_fifo_setup_sw(struct gk20a *g)
 464 {
 465         struct fifo_gk20a *f = &g->fifo;
 466         struct device *d = dev_from_gk20a(g);
 467         int chid, i, err = 0;
 468
 469         gk20a_dbg_fn("");
 470
 471         if (f->sw_ready) {
 472                 gk20a_dbg_fn("skip init");
 473                 return 0;
 474         }
 475
 476         f->g = g;
 477
 478         mutex_init(&f->intr.isr.mutex);
 479         mutex_init(&f->gr_reset_mutex);
 480         gk20a_init_fifo_pbdma_intr_descs(f); /* just filling in data/tables */
 481
 482         f->num_channels = g->ops.fifo.get_num_fifos(g);
 483         f->num_pbdma = proj_host_num_pbdma_v();
 484         f->max_engines = ENGINE_INVAL_GK20A;
 485
 486         f->userd_entry_size = 1 << ram_userd_base_shift_v();
 487
 488         err = gk20a_gmmu_alloc_map(&g->mm.bar1.vm,
 489                                    f->userd_entry_size * f->num_channels,
 490                                    &f->userd);
 491         if (err) {
 492                 dev_err(d, "memory allocation failed\n");
 493                 goto clean_up;
 494         }
 495
 496         gk20a_dbg(gpu_dbg_map, "userd bar1 va = 0x%llx", f->userd.gpu_va);
 497
 498         f->channel = kzalloc(f->num_channels * sizeof(*f->channel),
 499                                 GFP_KERNEL);
 500         f->tsg = kzalloc(f->num_channels * sizeof(*f->tsg),
 501                                 GFP_KERNEL);
 502         f->pbdma_map = kzalloc(f->num_pbdma * sizeof(*f->pbdma_map),
 503                                 GFP_KERNEL);
 504         f->engine_info = kzalloc(f->max_engines * sizeof(*f->engine_info),
 505                                 GFP_KERNEL);
 506
 507         if (!(f->channel && f->pbdma_map && f->engine_info)) {
 508                 err = -ENOMEM;
 509                 goto clean_up;
 510         }
 511
 512         /* pbdma map needs to be in place before calling engine info init */
 513         for (i = 0; i < f->num_pbdma; ++i)
 514                 f->pbdma_map[i] = gk20a_readl(g, fifo_pbdma_map_r(i));
 515
 516         init_engine_info(f);
 517
 518         init_runlist(g, f);
 519
 520         INIT_LIST_HEAD(&f->free_chs);
 521         mutex_init(&f->free_chs_mutex);
 522
 523         for (chid = 0; chid < f->num_channels; chid++) {
 524                 f->channel[chid].userd_cpu_va =
 525                         f->userd.cpu_va + chid * f->userd_entry_size;
 526                 f->channel[chid].userd_iova =
 527                         g->ops.mm.get_iova_addr(g, f->userd.sgt->sgl, 0)
 528                                 + chid * f->userd_entry_size;
 529                 f->channel[chid].userd_gpu_va =
 530                         f->userd.gpu_va + chid * f->userd_entry_size;
 531
 532                 gk20a_init_channel_support(g, chid);
 533                 gk20a_init_tsg_support(g, chid);
 534         }
 535         mutex_init(&f->tsg_inuse_mutex);
 536
 537         f->remove_support = gk20a_remove_fifo_support;
 538
 539         f->deferred_reset_pending = false;
 540         mutex_init(&f->deferred_reset_mutex);
 541
 542         f->sw_ready = true;
 543
 544         gk20a_dbg_fn("done");
 545         return 0;
 546
 547 clean_up:
 548         gk20a_dbg_fn("fail");
 549         gk20a_gmmu_unmap_free(&g->mm.bar1.vm, &f->userd);
 550
 551         kfree(f->channel);
 552         f->channel = NULL;
 553         kfree(f->pbdma_map);
 554         f->pbdma_map = NULL;
 555         kfree(f->engine_info);
 556         f->engine_info = NULL;
 557
 558         return err;
 559 }
 560
 561 static void gk20a_fifo_handle_runlist_event(struct gk20a *g)
 562 {
 563         u32 runlist_event = gk20a_readl(g, fifo_intr_runlist_r());
 564
 565         gk20a_dbg(gpu_dbg_intr, "runlist event %08x\n",
 566                   runlist_event);
 567
 568         gk20a_writel(g, fifo_intr_runlist_r(), runlist_event);
 569 }
 570
 571 static int gk20a_init_fifo_setup_hw(struct gk20a *g)
 572 {
 573         struct fifo_gk20a *f = &g->fifo;
 574
 575         gk20a_dbg_fn("");
 576
 577         /* test write, read through bar1 @ userd region before
 578          * turning on the snooping */
 579         {
 580                 struct fifo_gk20a *f = &g->fifo;
 581                 u32 v, v1 = 0x33, v2 = 0x55;
 582
 583                 u32 bar1_vaddr = f->userd.gpu_va;
 584                 volatile u32 *cpu_vaddr = f->userd.cpu_va;
 585
 586                 gk20a_dbg_info("test bar1 @ vaddr 0x%x",
 587                            bar1_vaddr);
 588
 589                 v = gk20a_bar1_readl(g, bar1_vaddr);
 590
 591                 *cpu_vaddr = v1;
 592                 smp_mb();
 593
 594                 if (v1 != gk20a_bar1_readl(g, bar1_vaddr)) {
 595                         gk20a_err(dev_from_gk20a(g), "bar1 broken @ gk20a: CPU wrote 0x%x, \
 596                                 GPU read 0x%x", *cpu_vaddr, gk20a_bar1_readl(g, bar1_vaddr));
 597                         return -EINVAL;
 598                 }
 599
 600                 gk20a_bar1_writel(g, bar1_vaddr, v2);
 601
 602                 if (v2 != gk20a_bar1_readl(g, bar1_vaddr)) {
 603                         gk20a_err(dev_from_gk20a(g), "bar1 broken @ gk20a: GPU wrote 0x%x, \
 604                                 CPU read 0x%x", gk20a_bar1_readl(g, bar1_vaddr), *cpu_vaddr);
 605                         return -EINVAL;
 606                 }
 607
 608                 /* is it visible to the cpu? */
 609                 if (*cpu_vaddr != v2) {
 610                         gk20a_err(dev_from_gk20a(g),
 611                                 "cpu didn't see bar1 write @ %p!",
 612                                 cpu_vaddr);
 613                 }
 614
 615                 /* put it back */
 616                 gk20a_bar1_writel(g, bar1_vaddr, v);
 617         }
 618
 619         /*XXX all manner of flushes and caching worries, etc */
 620
 621         /* set the base for the userd region now */
 622         gk20a_writel(g, fifo_bar1_base_r(),
 623                         fifo_bar1_base_ptr_f(f->userd.gpu_va >> 12) |
 624                         fifo_bar1_base_valid_true_f());
 625
 626         gk20a_dbg_fn("done");
 627
 628         return 0;
 629 }
 630
 631 int gk20a_init_fifo_support(struct gk20a *g)
 632 {
 633         u32 err;
 634
 635         err = gk20a_init_fifo_setup_sw(g);
 636         if (err)
 637                 return err;
 638
 639         err = gk20a_init_fifo_setup_hw(g);
 640         if (err)
 641                 return err;
 642
 643         return err;
 644 }
 645
 646 /* return with a reference to the channel, caller must put it back */
 647 static struct channel_gk20a *
 648 channel_from_inst_ptr(struct fifo_gk20a *f, u64 inst_ptr)
 649 {
 650         int ci;
 651         if (unlikely(!f->channel))
 652                 return NULL;
 653         for (ci = 0; ci < f->num_channels; ci++) {
 654                 struct channel_gk20a *ch = gk20a_channel_get(&f->channel[ci]);
 655                 /* only alive channels are searched */
 656                 if (!ch)
 657                         continue;
 658
 659                 if (ch->inst_block.cpu_va &&
 660                     (inst_ptr == gk20a_mem_phys(&ch->inst_block)))
 661                         return ch;
 662
 663                 gk20a_channel_put(ch);
 664         }
 665         return NULL;
 666 }
 667
 668 /* fault info/descriptions.
 669  * tbd: move to setup
 670  *  */
 671 static const char * const fault_type_descs[] = {
 672          "pde", /*fifo_intr_mmu_fault_info_type_pde_v() == 0 */
 673          "pde size",
 674          "pte",
 675          "va limit viol",
 676          "unbound inst",
 677          "priv viol",
 678          "ro viol",
 679          "wo viol",
 680          "pitch mask",
 681          "work creation",
 682          "bad aperture",
 683          "compression failure",
 684          "bad kind",
 685          "region viol",
 686          "dual ptes",
 687          "poisoned",
 688 };
 689 /* engine descriptions */
 690 static const char * const engine_subid_descs[] = {
 691         "gpc",
 692         "hub",
 693 };
 694
 695 static const char * const hub_client_descs[] = {
 696         "vip", "ce0", "ce1", "dniso", "fe", "fecs", "host", "host cpu",
 697         "host cpu nb", "iso", "mmu", "mspdec", "msppp", "msvld",
 698         "niso", "p2p", "pd", "perf", "pmu", "raster twod", "scc",
 699         "scc nb", "sec", "ssync", "gr copy", "ce2", "xv", "mmu nb",
 700         "msenc", "d falcon", "sked", "a falcon", "n/a",
 701 };
 702
 703 static const char * const gpc_client_descs[] = {
 704         "l1 0", "t1 0", "pe 0",
 705         "l1 1", "t1 1", "pe 1",
 706         "l1 2", "t1 2", "pe 2",
 707         "l1 3", "t1 3", "pe 3",
 708         "rast", "gcc", "gpccs",
 709         "prop 0", "prop 1", "prop 2", "prop 3",
 710         "l1 4", "t1 4", "pe 4",
 711         "l1 5", "t1 5", "pe 5",
 712         "l1 6", "t1 6", "pe 6",
 713         "l1 7", "t1 7", "pe 7",
 714         "gpm",
 715         "ltp utlb 0", "ltp utlb 1", "ltp utlb 2", "ltp utlb 3",
 716         "rgg utlb",
 717 };
 718
 719 /* reads info from hardware and fills in mmu fault info record */
 720 static inline void get_exception_mmu_fault_info(
 721         struct gk20a *g, u32 engine_id,
 722         struct fifo_mmu_fault_info_gk20a *f)
 723 {
 724         u32 fault_info_v;
 725
 726         gk20a_dbg_fn("engine_id %d", engine_id);
 727
 728         memset(f, 0, sizeof(*f));
 729
 730         f->fault_info_v = fault_info_v = gk20a_readl(g,
 731              fifo_intr_mmu_fault_info_r(engine_id));
 732         f->fault_type_v =
 733                 fifo_intr_mmu_fault_info_type_v(fault_info_v);
 734         f->engine_subid_v =
 735                 fifo_intr_mmu_fault_info_engine_subid_v(fault_info_v);
 736         f->client_v = fifo_intr_mmu_fault_info_client_v(fault_info_v);
 737
 738         BUG_ON(f->fault_type_v >= ARRAY_SIZE(fault_type_descs));
 739         f->fault_type_desc =  fault_type_descs[f->fault_type_v];
 740
 741         BUG_ON(f->engine_subid_v >= ARRAY_SIZE(engine_subid_descs));
 742         f->engine_subid_desc = engine_subid_descs[f->engine_subid_v];
 743
 744         if (f->engine_subid_v ==
 745             fifo_intr_mmu_fault_info_engine_subid_hub_v()) {
 746
 747                 BUG_ON(f->client_v >= ARRAY_SIZE(hub_client_descs));
 748                 f->client_desc = hub_client_descs[f->client_v];
 749         } else if (f->engine_subid_v ==
 750                    fifo_intr_mmu_fault_info_engine_subid_gpc_v()) {
 751                 BUG_ON(f->client_v >= ARRAY_SIZE(gpc_client_descs));
 752                 f->client_desc = gpc_client_descs[f->client_v];
 753         } else {
 754                 BUG_ON(1);
 755         }
 756
 757         f->fault_hi_v = gk20a_readl(g, fifo_intr_mmu_fault_hi_r(engine_id));
 758         f->fault_lo_v = gk20a_readl(g, fifo_intr_mmu_fault_lo_r(engine_id));
 759         /* note:ignoring aperture on gk20a... */
 760         f->inst_ptr = fifo_intr_mmu_fault_inst_ptr_v(
 761                  gk20a_readl(g, fifo_intr_mmu_fault_inst_r(engine_id)));
 762         /* note: inst_ptr is a 40b phys addr.  */
 763         f->inst_ptr <<= fifo_intr_mmu_fault_inst_ptr_align_shift_v();
 764 }
 765
 766 void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id)
 767 {
 768         gk20a_dbg_fn("");
 769
 770         if (engine_id == top_device_info_type_enum_graphics_v()) {
 771                 if (support_gk20a_pmu(g->dev) && g->elpg_enabled)
 772                         gk20a_pmu_disable_elpg(g);
 773                         /*HALT_PIPELINE method, halt GR engine*/
 774                         if (gr_gk20a_halt_pipe(g))
 775                                 gk20a_err(dev_from_gk20a(g),
 776                                         "failed to HALT gr pipe");
 777                         /* resetting engine using mc_enable_r() is not
 778                         enough, we do full init sequence */
 779                         gk20a_gr_reset(g);
 780                         gk20a_pmu_enable_elpg(g);
 781         }
 782         if (engine_id == top_device_info_type_enum_copy0_v())
 783                 gk20a_reset(g, mc_enable_ce2_m());
 784 }
 785
 786 static void gk20a_fifo_handle_chsw_fault(struct gk20a *g)
 787 {
 788         u32 intr;
 789
 790         intr = gk20a_readl(g, fifo_intr_chsw_error_r());
 791         gk20a_err(dev_from_gk20a(g), "chsw: %08x\n", intr);
 792         gk20a_fecs_dump_falcon_stats(g);
 793         gk20a_writel(g, fifo_intr_chsw_error_r(), intr);
 794 }
 795
 796 static void gk20a_fifo_handle_dropped_mmu_fault(struct gk20a *g)
 797 {
 798         struct device *dev = dev_from_gk20a(g);
 799         u32 fault_id = gk20a_readl(g, fifo_intr_mmu_fault_id_r());
 800         gk20a_err(dev, "dropped mmu fault (0x%08x)", fault_id);
 801 }
 802
 803 static bool gk20a_fifo_should_defer_engine_reset(struct gk20a *g, u32 engine_id,
 804                 struct fifo_mmu_fault_info_gk20a *f, bool fake_fault)
 805 {
 806         /* channel recovery is only deferred if an sm debugger
 807            is attached and has MMU debug mode is enabled */
 808         if (!gk20a_gr_sm_debugger_attached(g) ||
 809             !g->ops.mm.is_debug_mode_enabled(g))
 810                 return false;
 811
 812         /* if this fault is fake (due to RC recovery), don't defer recovery */
 813         if (fake_fault)
 814                 return false;
 815
 816         if (engine_id != ENGINE_GR_GK20A ||
 817             f->engine_subid_v != fifo_intr_mmu_fault_info_engine_subid_gpc_v())
 818                 return false;
 819
 820         return true;
 821 }
 822
 823 /* caller must hold a channel reference */
 824 static bool gk20a_fifo_set_ctx_mmu_error(struct gk20a *g,
 825                 struct channel_gk20a *ch)
 826 {
 827         bool verbose = true;
 828         if (!ch)
 829                 return verbose;
 830
 831         if (ch->error_notifier) {
 832                 u32 err = ch->error_notifier->info32;
 833                 if (ch->error_notifier->status == 0xffff) {
 834                         /* If error code is already set, this mmu fault
 835                          * was triggered as part of recovery from other
 836                          * error condition.
 837                          * Don't overwrite error flag. */
 838                         /* Fifo timeout debug spew is controlled by user */
 839                         if (err == NVGPU_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT)
 840                                 verbose = ch->timeout_debug_dump;
 841                 } else {
 842                         gk20a_set_error_notifier(ch,
 843                                 NVGPU_CHANNEL_FIFO_ERROR_MMU_ERR_FLT);
 844                 }
 845         }
 846         /* mark channel as faulted */
 847         ch->has_timedout = true;
 848         wmb();
 849         /* unblock pending waits */
 850         wake_up(&ch->semaphore_wq);
 851         wake_up(&ch->notifier_wq);
 852         wake_up(&ch->submit_wq);
 853         return verbose;
 854 }
 855
 856 static bool gk20a_fifo_set_ctx_mmu_error_ch(struct gk20a *g,
 857                 struct channel_gk20a *ch)
 858 {
 859         gk20a_err(dev_from_gk20a(g),
 860                 "channel %d generated a mmu fault", ch->hw_chid);
 861
 862         return gk20a_fifo_set_ctx_mmu_error(g, ch);
 863 }
 864
 865 static bool gk20a_fifo_set_ctx_mmu_error_tsg(struct gk20a *g,
 866                 struct tsg_gk20a *tsg)
 867 {
 868         bool ret = true;
 869         struct channel_gk20a *ch = NULL;
 870
 871         gk20a_err(dev_from_gk20a(g),
 872                 "TSG %d generated a mmu fault", tsg->tsgid);
 873
 874         mutex_lock(&tsg->ch_list_lock);
 875         list_for_each_entry(ch, &tsg->ch_list, ch_entry) {
 876                 if (gk20a_channel_get(ch)) {
 877                         if (!gk20a_fifo_set_ctx_mmu_error(g, ch))
 878                                 ret = false;
 879                         gk20a_channel_put(ch);
 880                 }
 881         }
 882         mutex_unlock(&tsg->ch_list_lock);
 883
 884         return ret;
 885 }
 886
 887 static void gk20a_fifo_abort_tsg(struct gk20a *g, u32 tsgid)
 888 {
 889         struct tsg_gk20a *tsg = &g->fifo.tsg[tsgid];
 890         struct channel_gk20a *ch;
 891
 892         mutex_lock(&tsg->ch_list_lock);
 893         list_for_each_entry(ch, &tsg->ch_list, ch_entry) {
 894                 if (gk20a_channel_get(ch)) {
 895                         gk20a_channel_abort(ch, false);
 896                         gk20a_channel_put(ch);
 897                 }
 898         }
 899         mutex_unlock(&tsg->ch_list_lock);
 900 }
 901
 902 static bool gk20a_fifo_handle_mmu_fault(
 903         struct gk20a *g,
 904         u32 mmu_fault_engines, /* queried from HW if 0 */
 905         u32 hw_id, /* queried from HW if ~(u32)0 OR mmu_fault_engines == 0*/
 906         bool id_is_tsg)
 907 {
 908         bool fake_fault;
 909         unsigned long fault_id;
 910         unsigned long engine_mmu_id;
 911         bool verbose = true;
 912         u32 grfifo_ctl;
 913
 914         gk20a_dbg_fn("");
 915
 916         g->fifo.deferred_reset_pending = false;
 917
 918         /* Disable power management */
 919         if (support_gk20a_pmu(g->dev) && g->elpg_enabled)
 920                 gk20a_pmu_disable_elpg(g);
 921         g->ops.clock_gating.slcg_gr_load_gating_prod(g,
 922                         false);
 923         g->ops.clock_gating.slcg_perf_load_gating_prod(g,
 924                         false);
 925         g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
 926                         false);
 927         gr_gk20a_init_elcg_mode(g, ELCG_RUN, ENGINE_GR_GK20A);
 928         gr_gk20a_init_elcg_mode(g, ELCG_RUN, ENGINE_CE2_GK20A);
 929
 930         /* Disable fifo access */
 931         grfifo_ctl = gk20a_readl(g, gr_gpfifo_ctl_r());
 932         grfifo_ctl &= ~gr_gpfifo_ctl_semaphore_access_f(1);
 933         grfifo_ctl &= ~gr_gpfifo_ctl_access_f(1);
 934
 935         gk20a_writel(g, gr_gpfifo_ctl_r(),
 936                 grfifo_ctl | gr_gpfifo_ctl_access_f(0) |
 937                 gr_gpfifo_ctl_semaphore_access_f(0));
 938
 939         if (mmu_fault_engines) {
 940                 fault_id = mmu_fault_engines;
 941                 fake_fault = true;
 942         } else {
 943                 fault_id = gk20a_readl(g, fifo_intr_mmu_fault_id_r());
 944                 fake_fault = false;
 945                 gk20a_debug_dump(g->dev);
 946         }
 947
 948
 949         /* go through all faulted engines */
 950         for_each_set_bit(engine_mmu_id, &fault_id, 32) {
 951                 /* bits in fifo_intr_mmu_fault_id_r do not correspond 1:1 to
 952                  * engines. Convert engine_mmu_id to engine_id */
 953                 u32 engine_id = gk20a_mmu_id_to_engine_id(engine_mmu_id);
 954                 struct fifo_mmu_fault_info_gk20a f;
 955                 struct channel_gk20a *ch = NULL;
 956                 struct tsg_gk20a *tsg = NULL;
 957                 struct channel_gk20a *referenced_channel = NULL;
 958                 bool was_reset;
 959                 /* read and parse engine status */
 960                 u32 status = gk20a_readl(g, fifo_engine_status_r(engine_id));
 961                 u32 ctx_status = fifo_engine_status_ctx_status_v(status);
 962                 bool ctxsw = (ctx_status ==
 963                                 fifo_engine_status_ctx_status_ctxsw_switch_v()
 964                                 || ctx_status ==
 965                                 fifo_engine_status_ctx_status_ctxsw_save_v()
 966                                 || ctx_status ==
 967                                 fifo_engine_status_ctx_status_ctxsw_load_v());
 968
 969                 get_exception_mmu_fault_info(g, engine_mmu_id, &f);
 970                 trace_gk20a_mmu_fault(f.fault_hi_v,
 971                                       f.fault_lo_v,
 972                                       f.fault_info_v,
 973                                       f.inst_ptr,
 974                                       engine_id,
 975                                       f.engine_subid_desc,
 976                                       f.client_desc,
 977                                       f.fault_type_desc);
 978                 gk20a_err(dev_from_gk20a(g), "mmu fault on engine %d, "
 979                            "engine subid %d (%s), client %d (%s), "
 980                            "addr 0x%08x:0x%08x, type %d (%s), info 0x%08x,"
 981                            "inst_ptr 0x%llx\n",
 982                            engine_id,
 983                            f.engine_subid_v, f.engine_subid_desc,
 984                            f.client_v, f.client_desc,
 985                            f.fault_hi_v, f.fault_lo_v,
 986                            f.fault_type_v, f.fault_type_desc,
 987                            f.fault_info_v, f.inst_ptr);
 988
 989                 if (ctxsw) {
 990                         gk20a_fecs_dump_falcon_stats(g);
 991                         gk20a_err(dev_from_gk20a(g), "gr_status_r : 0x%x",
 992                                         gk20a_readl(g, gr_status_r()));
 993                 }
 994
 995                 /* get the channel/TSG */
 996                 if (fake_fault) {
 997                         /* use next_id if context load is failing */
 998                         u32 id, type;
 999
1000                         if (hw_id == ~(u32)0) {
1001                                 id = (ctx_status ==
1002                                       fifo_engine_status_ctx_status_ctxsw_load_v()) ?
1003                                         fifo_engine_status_next_id_v(status) :
1004                                         fifo_engine_status_id_v(status);
1005                                 type = (ctx_status ==
1006                                         fifo_engine_status_ctx_status_ctxsw_load_v()) ?
1007                                         fifo_engine_status_next_id_type_v(status) :
1008                                         fifo_engine_status_id_type_v(status);
1009                         } else {
1010                                 id = hw_id;
1011                                 type = id_is_tsg ?
1012                                         fifo_engine_status_id_type_tsgid_v() :
1013                                         fifo_engine_status_id_type_chid_v();
1014                         }
1015
1016                         if (type == fifo_engine_status_id_type_tsgid_v())
1017                                 tsg = &g->fifo.tsg[id];
1018                         else if (type == fifo_engine_status_id_type_chid_v()) {
1019                                 ch = &g->fifo.channel[id];
1020                                 referenced_channel = gk20a_channel_get(ch);
1021                         }
1022                 } else {
1023                         /* read channel based on instruction pointer */
1024                         ch = channel_from_inst_ptr(&g->fifo, f.inst_ptr);
1025                         referenced_channel = ch;
1026                 }
1027
1028                 if (ch && gk20a_is_channel_marked_as_tsg(ch))
1029                         tsg = &g->fifo.tsg[ch->tsgid];
1030
1031                 /* check if engine reset should be deferred */
1032                 if ((ch || tsg) && gk20a_fifo_should_defer_engine_reset(g,
1033                                 engine_id, &f, fake_fault)) {
1034                         g->fifo.deferred_fault_engines = fault_id;
1035
1036                         /* handled during channel free */
1037                         g->fifo.deferred_reset_pending = true;
1038                 } else if (engine_id != ~0) {
1039                         was_reset = mutex_is_locked(&g->fifo.gr_reset_mutex);
1040                         mutex_lock(&g->fifo.gr_reset_mutex);
1041                         /* if lock is already taken, a reset is taking place
1042                         so no need to repeat */
1043                         if (!was_reset)
1044                                 gk20a_fifo_reset_engine(g, engine_id);
1045                         mutex_unlock(&g->fifo.gr_reset_mutex);
1046                 }
1047                 /* disable the channel/TSG from hw and increment
1048                  * syncpoints */
1049
1050                 if (tsg) {
1051                         if (!g->fifo.deferred_reset_pending)
1052                                 verbose =
1053                                        gk20a_fifo_set_ctx_mmu_error_tsg(g, tsg);
1054
1055                         gk20a_fifo_abort_tsg(g, tsg->tsgid);
1056
1057                         /* put back the ref taken early above */
1058                         if (referenced_channel)
1059                                 gk20a_channel_put(ch);
1060                 } else if (ch) {
1061                         if (referenced_channel) {
1062                                 if (!g->fifo.deferred_reset_pending)
1063                                         verbose = gk20a_fifo_set_ctx_mmu_error_ch(g, ch);
1064                                 gk20a_channel_abort(ch, false);
1065                                 gk20a_channel_put(ch);
1066                         } else {
1067                                 gk20a_err(dev_from_gk20a(g),
1068                                                 "mmu error in freed channel %d",
1069                                                 ch->hw_chid);
1070                         }
1071                 } else if (f.inst_ptr ==
1072                                 gk20a_mem_phys(&g->mm.bar1.inst_block)) {
1073                         gk20a_err(dev_from_gk20a(g), "mmu fault from bar1");
1074                 } else if (f.inst_ptr ==
1075                                 gk20a_mem_phys(&g->mm.pmu.inst_block)) {
1076                         gk20a_err(dev_from_gk20a(g), "mmu fault from pmu");
1077                 } else
1078                         gk20a_err(dev_from_gk20a(g), "couldn't locate channel for mmu fault");
1079         }
1080
1081         if (g->fifo.deferred_reset_pending) {
1082                 gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, "sm debugger attached,"
1083                            " deferring channel recovery to channel free");
1084                 /* clear interrupt */
1085                 gk20a_writel(g, fifo_intr_mmu_fault_id_r(), fault_id);
1086                 goto exit_enable;
1087         }
1088
1089         /* clear interrupt */
1090         gk20a_writel(g, fifo_intr_mmu_fault_id_r(), fault_id);
1091
1092         /* resume scheduler */
1093         gk20a_writel(g, fifo_error_sched_disable_r(),
1094                      gk20a_readl(g, fifo_error_sched_disable_r()));
1095
1096         /* Re-enable fifo access */
1097         gk20a_writel(g, gr_gpfifo_ctl_r(),
1098                      gr_gpfifo_ctl_access_enabled_f() |
1099                      gr_gpfifo_ctl_semaphore_access_enabled_f());
1100
1101 exit_enable:
1102         /* It is safe to enable ELPG again. */
1103         if (support_gk20a_pmu(g->dev) && g->elpg_enabled)
1104                 gk20a_pmu_enable_elpg(g);
1105         return verbose;
1106 }
1107
1108 static void gk20a_fifo_get_faulty_id_type(struct gk20a *g, int engine_id,
1109                                           u32 *id, u32 *type)
1110 {
1111         u32 status = gk20a_readl(g, fifo_engine_status_r(engine_id));
1112         u32 ctx_status = fifo_engine_status_ctx_status_v(status);
1113
1114         /* use next_id if context load is failing */
1115         *id = (ctx_status ==
1116                 fifo_engine_status_ctx_status_ctxsw_load_v()) ?
1117                 fifo_engine_status_next_id_v(status) :
1118                 fifo_engine_status_id_v(status);
1119
1120         *type = (ctx_status ==
1121                 fifo_engine_status_ctx_status_ctxsw_load_v()) ?
1122                 fifo_engine_status_next_id_type_v(status) :
1123                 fifo_engine_status_id_type_v(status);
1124 }
1125
1126 static void gk20a_fifo_trigger_mmu_fault(struct gk20a *g,
1127                 unsigned long engine_ids)
1128 {
1129         unsigned long end_jiffies = jiffies +
1130                 msecs_to_jiffies(gk20a_get_gr_idle_timeout(g));
1131         unsigned long delay = GR_IDLE_CHECK_DEFAULT;
1132         unsigned long engine_id;
1133         int ret;
1134
1135         /* trigger faults for all bad engines */
1136         for_each_set_bit(engine_id, &engine_ids, 32) {
1137                 if (engine_id > g->fifo.max_engines) {
1138                         WARN_ON(true);
1139                         break;
1140                 }
1141
1142                 gk20a_writel(g, fifo_trigger_mmu_fault_r(engine_id),
1143                              fifo_trigger_mmu_fault_id_f(
1144                              gk20a_engine_id_to_mmu_id(engine_id)) |
1145                              fifo_trigger_mmu_fault_enable_f(1));
1146         }
1147
1148         /* Wait for MMU fault to trigger */
1149         ret = -EBUSY;
1150         do {
1151                 if (gk20a_readl(g, fifo_intr_0_r()) &
1152                                 fifo_intr_0_mmu_fault_pending_f()) {
1153                         ret = 0;
1154                         break;
1155                 }
1156
1157                 usleep_range(delay, delay * 2);
1158                 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
1159         } while (time_before(jiffies, end_jiffies) ||
1160                         !tegra_platform_is_silicon());
1161
1162         if (ret)
1163                 gk20a_err(dev_from_gk20a(g), "mmu fault timeout");
1164
1165         /* release mmu fault trigger */
1166         for_each_set_bit(engine_id, &engine_ids, 32)
1167                 gk20a_writel(g, fifo_trigger_mmu_fault_r(engine_id), 0);
1168 }
1169
1170 static u32 gk20a_fifo_engines_on_id(struct gk20a *g, u32 id, bool is_tsg)
1171 {
1172         int i;
1173         u32 engines = 0;
1174
1175         for (i = 0; i < g->fifo.max_engines; i++) {
1176                 u32 status = gk20a_readl(g, fifo_engine_status_r(i));
1177                 u32 ctx_status =
1178                         fifo_engine_status_ctx_status_v(status);
1179                 u32 ctx_id = (ctx_status ==
1180                         fifo_engine_status_ctx_status_ctxsw_load_v()) ?
1181                         fifo_engine_status_next_id_v(status) :
1182                         fifo_engine_status_id_v(status);
1183                 u32 type = (ctx_status ==
1184                         fifo_engine_status_ctx_status_ctxsw_load_v()) ?
1185                         fifo_engine_status_next_id_type_v(status) :
1186                         fifo_engine_status_id_type_v(status);
1187                 bool busy = fifo_engine_status_engine_v(status) ==
1188                         fifo_engine_status_engine_busy_v();
1189                 if (busy && ctx_id == id) {
1190                         if ((is_tsg && type ==
1191                                         fifo_engine_status_id_type_tsgid_v()) ||
1192                                     (!is_tsg && type ==
1193                                         fifo_engine_status_id_type_chid_v()))
1194                                 engines |= BIT(i);
1195                 }
1196         }
1197
1198         return engines;
1199 }
1200
1201 void gk20a_fifo_recover_ch(struct gk20a *g, u32 hw_chid, bool verbose)
1202 {
1203         u32 engines;
1204
1205         /* stop context switching to prevent engine assignments from
1206            changing until channel is recovered */
1207         mutex_lock(&g->dbg_sessions_lock);
1208         gr_gk20a_disable_ctxsw(g);
1209
1210         engines = gk20a_fifo_engines_on_id(g, hw_chid, false);
1211
1212         if (engines)
1213                 gk20a_fifo_recover(g, engines, hw_chid, false, true, verbose);
1214         else {
1215                 struct channel_gk20a *ch = &g->fifo.channel[hw_chid];
1216
1217                 if (gk20a_channel_get(ch)) {
1218                         gk20a_channel_abort(ch, false);
1219
1220                         if (gk20a_fifo_set_ctx_mmu_error_ch(g, ch))
1221                                 gk20a_debug_dump(g->dev);
1222
1223                         gk20a_channel_put(ch);
1224                 }
1225         }
1226
1227         gr_gk20a_enable_ctxsw(g);
1228         mutex_unlock(&g->dbg_sessions_lock);
1229 }
1230
1231 void gk20a_fifo_recover_tsg(struct gk20a *g, u32 tsgid, bool verbose)
1232 {
1233         u32 engines;
1234
1235         /* stop context switching to prevent engine assignments from
1236            changing until TSG is recovered */
1237         mutex_lock(&g->dbg_sessions_lock);
1238         gr_gk20a_disable_ctxsw(g);
1239
1240         engines = gk20a_fifo_engines_on_id(g, tsgid, true);
1241
1242         if (engines)
1243                 gk20a_fifo_recover(g, engines, tsgid, true, true, verbose);
1244         else {
1245                 struct tsg_gk20a *tsg = &g->fifo.tsg[tsgid];
1246
1247                 if (gk20a_fifo_set_ctx_mmu_error_tsg(g, tsg))
1248                         gk20a_debug_dump(g->dev);
1249
1250                 gk20a_fifo_abort_tsg(g, tsgid);
1251         }
1252
1253         gr_gk20a_enable_ctxsw(g);
1254         mutex_unlock(&g->dbg_sessions_lock);
1255 }
1256
1257 void gk20a_fifo_recover(struct gk20a *g, u32 __engine_ids,
1258                         u32 hw_id, bool id_is_tsg,
1259                         bool id_is_known, bool verbose)
1260 {
1261         unsigned long engine_id, i;
1262         unsigned long _engine_ids = __engine_ids;
1263         unsigned long engine_ids = 0;
1264         u32 val;
1265         u32 mmu_fault_engines = 0;
1266         u32 ref_type;
1267         u32 ref_id;
1268         u32 ref_id_is_tsg = false;
1269
1270         if (verbose)
1271                 gk20a_debug_dump(g->dev);
1272
1273         if (g->ops.ltc.flush)
1274                 g->ops.ltc.flush(g);
1275
1276         if (id_is_known) {
1277                 engine_ids = gk20a_fifo_engines_on_id(g, hw_id, id_is_tsg);
1278                 ref_id = hw_id;
1279                 ref_type = id_is_tsg ?
1280                         fifo_engine_status_id_type_tsgid_v() :
1281                         fifo_engine_status_id_type_chid_v();
1282                 ref_id_is_tsg = id_is_tsg;
1283                 /* atleast one engine will get passed during sched err*/
1284                 engine_ids |= __engine_ids;
1285                 for_each_set_bit(engine_id, &engine_ids, 32) {
1286                         mmu_fault_engines |=
1287                                 BIT(gk20a_engine_id_to_mmu_id(engine_id));
1288                 }
1289         } else {
1290                 /* store faulted engines in advance */
1291                 for_each_set_bit(engine_id, &_engine_ids, 32) {
1292                         gk20a_fifo_get_faulty_id_type(g, engine_id, &ref_id,
1293                                                       &ref_type);
1294                         if (ref_type == fifo_engine_status_id_type_tsgid_v())
1295                                 ref_id_is_tsg = true;
1296                         else
1297                                 ref_id_is_tsg = false;
1298                         /* Reset *all* engines that use the
1299                          * same channel as faulty engine */
1300                         for (i = 0; i < g->fifo.max_engines; i++) {
1301                                 u32 type;
1302                                 u32 id;
1303                                 gk20a_fifo_get_faulty_id_type(g, i, &id, &type);
1304                                 if (ref_type == type && ref_id == id) {
1305                                         engine_ids |= BIT(i);
1306                                         mmu_fault_engines |=
1307                                         BIT(gk20a_engine_id_to_mmu_id(i));
1308                                 }
1309                         }
1310                 }
1311         }
1312
1313         if (mmu_fault_engines) {
1314                 /*
1315                  * sched error prevents recovery, and ctxsw error will retrigger
1316                  * every 100ms. Disable the sched error to allow recovery.
1317                  */
1318                 val = gk20a_readl(g, fifo_intr_en_0_r());
1319                 val &= ~(fifo_intr_en_0_sched_error_m() |
1320                         fifo_intr_en_0_mmu_fault_m());
1321                 gk20a_writel(g, fifo_intr_en_0_r(), val);
1322                 gk20a_writel(g, fifo_intr_0_r(),
1323                                 fifo_intr_0_sched_error_reset_f());
1324
1325                 g->ops.fifo.trigger_mmu_fault(g, engine_ids);
1326                 gk20a_fifo_handle_mmu_fault(g, mmu_fault_engines, ref_id,
1327                                 ref_id_is_tsg);
1328
1329                 val = gk20a_readl(g, fifo_intr_en_0_r());
1330                 val |= fifo_intr_en_0_mmu_fault_f(1)
1331                         | fifo_intr_en_0_sched_error_f(1);
1332                 gk20a_writel(g, fifo_intr_en_0_r(), val);
1333         }
1334 }
1335
1336 /* force reset channel and tsg (if it's part of one) */
1337 int gk20a_fifo_force_reset_ch(struct channel_gk20a *ch, bool verbose)
1338 {
1339         struct tsg_gk20a *tsg = NULL;
1340         struct channel_gk20a *ch_tsg = NULL;
1341         struct gk20a *g = ch->g;
1342
1343         if (gk20a_is_channel_marked_as_tsg(ch)) {
1344                 tsg = &g->fifo.tsg[ch->hw_chid];
1345
1346                 mutex_lock(&tsg->ch_list_lock);
1347
1348                 list_for_each_entry(ch_tsg, &tsg->ch_list, ch_entry) {
1349                         if (gk20a_channel_get(ch_tsg)) {
1350                                 gk20a_set_error_notifier(ch_tsg,
1351                                        NVGPU_CHANNEL_RESETCHANNEL_VERIF_ERROR);
1352                                 gk20a_channel_put(ch_tsg);
1353                         }
1354                 }
1355
1356                 mutex_unlock(&tsg->ch_list_lock);
1357                 gk20a_fifo_recover_tsg(g, ch->tsgid, verbose);
1358         } else {
1359                 gk20a_set_error_notifier(ch,
1360                         NVGPU_CHANNEL_RESETCHANNEL_VERIF_ERROR);
1361                 gk20a_fifo_recover_ch(g, ch->hw_chid, verbose);
1362         }
1363
1364         return 0;
1365 }
1366
1367 static bool gk20a_fifo_handle_sched_error(struct gk20a *g)
1368 {
1369         u32 sched_error;
1370         u32 engine_id;
1371         int id = -1;
1372         bool non_chid = false;
1373         bool ret = false;
1374         u32 mailbox2;
1375         /* read the scheduler error register */
1376         sched_error = gk20a_readl(g, fifo_intr_sched_error_r());
1377
1378         for (engine_id = 0; engine_id < g->fifo.max_engines; engine_id++) {
1379                 u32 status = gk20a_readl(g, fifo_engine_status_r(engine_id));
1380                 u32 ctx_status = fifo_engine_status_ctx_status_v(status);
1381                 bool failing_engine;
1382
1383                 /* we are interested in busy engines */
1384                 failing_engine = fifo_engine_status_engine_v(status) ==
1385                         fifo_engine_status_engine_busy_v();
1386
1387                 /* ..that are doing context switch */
1388                 failing_engine = failing_engine &&
1389                         (ctx_status ==
1390                                 fifo_engine_status_ctx_status_ctxsw_switch_v()
1391                         || ctx_status ==
1392                                 fifo_engine_status_ctx_status_ctxsw_save_v()
1393                         || ctx_status ==
1394                                 fifo_engine_status_ctx_status_ctxsw_load_v());
1395
1396                 if (!failing_engine)
1397                         continue;
1398                 if (ctx_status ==
1399                 fifo_engine_status_ctx_status_ctxsw_load_v()) {
1400                         id = fifo_engine_status_next_id_v(status);
1401                         non_chid = fifo_pbdma_status_id_type_v(status)
1402                                 != fifo_pbdma_status_id_type_chid_v();
1403                 } else if (ctx_status ==
1404                 fifo_engine_status_ctx_status_ctxsw_switch_v()) {
1405                         mailbox2 = gk20a_readl(g, gr_fecs_ctxsw_mailbox_r(2));
1406                         if (mailbox2 & FECS_METHOD_WFI_RESTORE)
1407                                 id = fifo_engine_status_next_id_v(status);
1408                         else
1409                                 id = fifo_engine_status_id_v(status);
1410                 } else {
1411                         id = fifo_engine_status_id_v(status);
1412                 }
1413                 break;
1414         }
1415
1416         /* could not find the engine - should never happen */
1417         if (unlikely(engine_id >= g->fifo.max_engines)) {
1418                 gk20a_err(dev_from_gk20a(g), "fifo sched error : 0x%08x, failed to find engine\n",
1419                         sched_error);
1420                 ret = false;
1421                 goto err;
1422         }
1423
1424         if (fifo_intr_sched_error_code_f(sched_error) ==
1425                         fifo_intr_sched_error_code_ctxsw_timeout_v()) {
1426                 struct fifo_gk20a *f = &g->fifo;
1427                 struct channel_gk20a *ch = &f->channel[id];
1428
1429                 if (non_chid) {
1430                         gk20a_fifo_recover(g, BIT(engine_id), id, true,
1431                                         true, true);
1432                         ret = true;
1433                         goto err;
1434                 }
1435
1436                 if (!gk20a_channel_get(ch))
1437                         goto err;
1438
1439                 if (gk20a_channel_update_and_check_timeout(ch,
1440                         GRFIFO_TIMEOUT_CHECK_PERIOD_US / 1000)) {
1441                         gk20a_set_error_notifier(ch,
1442                                 NVGPU_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT);
1443                         gk20a_err(dev_from_gk20a(g),
1444                                 "fifo sched ctxsw timeout error:"
1445                                 "engine = %u, ch = %d", engine_id, id);
1446                         gk20a_gr_debug_dump(g->dev);
1447                         gk20a_fifo_recover(g, BIT(engine_id), id, false,
1448                                 true, ch->timeout_debug_dump);
1449                         ret = true;
1450                 } else {
1451                         gk20a_dbg_info(
1452                                 "fifo is waiting for ctx switch for %d ms,"
1453                                 "ch = %d\n",
1454                                 ch->timeout_accumulated_ms,
1455                                 id);
1456                         ret = false;
1457                 }
1458                 gk20a_channel_put(ch);
1459                 return ret;
1460         }
1461
1462         gk20a_err(dev_from_gk20a(g), "fifo sched error : 0x%08x, engine=%u, %s=%d",
1463                 sched_error, engine_id, non_chid ? "non-ch" : "ch", id);
1464
1465 err:
1466         return ret;
1467 }
1468
1469 static u32 fifo_error_isr(struct gk20a *g, u32 fifo_intr)
1470 {
1471         bool print_channel_reset_log = false;
1472         struct device *dev = dev_from_gk20a(g);
1473         u32 handled = 0;
1474
1475         gk20a_dbg_fn("");
1476
1477         if (fifo_intr & fifo_intr_0_pio_error_pending_f()) {
1478                 /* pio mode is unused.  this shouldn't happen, ever. */
1479                 /* should we clear it or just leave it pending? */
1480                 gk20a_err(dev, "fifo pio error!\n");
1481                 BUG_ON(1);
1482         }
1483
1484         if (fifo_intr & fifo_intr_0_bind_error_pending_f()) {
1485                 u32 bind_error = gk20a_readl(g, fifo_intr_bind_error_r());
1486                 gk20a_err(dev, "fifo bind error: 0x%08x", bind_error);
1487                 print_channel_reset_log = true;
1488                 handled |= fifo_intr_0_bind_error_pending_f();
1489         }
1490
1491         if (fifo_intr & fifo_intr_0_sched_error_pending_f()) {
1492                 print_channel_reset_log = gk20a_fifo_handle_sched_error(g);
1493                 handled |= fifo_intr_0_sched_error_pending_f();
1494         }
1495
1496         if (fifo_intr & fifo_intr_0_chsw_error_pending_f()) {
1497                 gk20a_fifo_handle_chsw_fault(g);
1498                 handled |= fifo_intr_0_chsw_error_pending_f();
1499         }
1500
1501         if (fifo_intr & fifo_intr_0_mmu_fault_pending_f()) {
1502                 print_channel_reset_log =
1503                         gk20a_fifo_handle_mmu_fault(g, 0, ~(u32)0, false);
1504                 handled |= fifo_intr_0_mmu_fault_pending_f();
1505         }
1506
1507         if (fifo_intr & fifo_intr_0_dropped_mmu_fault_pending_f()) {
1508                 gk20a_fifo_handle_dropped_mmu_fault(g);
1509                 handled |= fifo_intr_0_dropped_mmu_fault_pending_f();
1510         }
1511
1512         print_channel_reset_log = !g->fifo.deferred_reset_pending
1513                         && print_channel_reset_log;
1514
1515         if (print_channel_reset_log) {
1516                 int engine_id;
1517                 gk20a_err(dev_from_gk20a(g),
1518                            "channel reset initiated from %s; intr=0x%08x",
1519                            __func__, fifo_intr);
1520                 for (engine_id = 0;
1521                      engine_id < g->fifo.max_engines;
1522                      engine_id++) {
1523                         gk20a_dbg_fn("enum:%d -> engine_id:%d", engine_id,
1524                                 g->fifo.engine_info[engine_id].engine_id);
1525                         fifo_pbdma_exception_status(g,
1526                                         &g->fifo.engine_info[engine_id]);
1527                         fifo_engine_exception_status(g,
1528                                         &g->fifo.engine_info[engine_id]);
1529                 }
1530         }
1531
1532         return handled;
1533 }
1534
1535 static inline void gk20a_fifo_reset_pbdma_header(struct gk20a *g, int pbdma_id)
1536 {
1537         gk20a_writel(g, pbdma_pb_header_r(pbdma_id),
1538                         pbdma_pb_header_first_true_f() |
1539                         pbdma_pb_header_type_non_inc_f());
1540 }
1541
1542 static inline void gk20a_fifo_reset_pbdma_method(struct gk20a *g, int pbdma_id,
1543                                                 int pbdma_method_index)
1544 {
1545         u32 pbdma_method_stride;
1546         u32 pbdma_method_reg;
1547
1548         pbdma_method_stride = pbdma_method1_r(pbdma_id) -
1549                                 pbdma_method0_r(pbdma_id);
1550
1551         pbdma_method_reg = pbdma_method0_r(pbdma_id) +
1552                 (pbdma_method_index * pbdma_method_stride);
1553
1554         gk20a_writel(g, pbdma_method_reg,
1555                         pbdma_method0_valid_true_f() |
1556                         pbdma_method0_first_true_f() |
1557                         pbdma_method0_addr_f(
1558                              pbdma_udma_nop_r() >> 2));
1559 }
1560
1561 static bool gk20a_fifo_is_sw_method_subch(struct gk20a *g, int pbdma_id,
1562                                                 int pbdma_method_index)
1563 {
1564         u32 pbdma_method_stride;
1565         u32 pbdma_method_reg, pbdma_method_subch;
1566
1567         pbdma_method_stride = pbdma_method1_r(pbdma_id) -
1568                                 pbdma_method0_r(pbdma_id);
1569
1570         pbdma_method_reg = pbdma_method0_r(pbdma_id) +
1571                         (pbdma_method_index * pbdma_method_stride);
1572
1573         pbdma_method_subch = pbdma_method0_subch_v(
1574                         gk20a_readl(g, pbdma_method_reg));
1575
1576         if (pbdma_method_subch == 5 || pbdma_method_subch == 6 ||
1577                                        pbdma_method_subch == 7)
1578                 return true;
1579
1580         return false;
1581 }
1582
1583 static u32 gk20a_fifo_handle_pbdma_intr(struct device *dev,
1584                                         struct gk20a *g,
1585                                         struct fifo_gk20a *f,
1586                                         u32 pbdma_id)
1587 {
1588         u32 pbdma_intr_0 = gk20a_readl(g, pbdma_intr_0_r(pbdma_id));
1589         u32 pbdma_intr_1 = gk20a_readl(g, pbdma_intr_1_r(pbdma_id));
1590         u32 handled = 0;
1591         bool reset = false;
1592         int i;
1593
1594         gk20a_dbg_fn("");
1595
1596         gk20a_dbg(gpu_dbg_intr, "pbdma id intr pending %d %08x %08x", pbdma_id,
1597                         pbdma_intr_0, pbdma_intr_1);
1598         if (pbdma_intr_0) {
1599                 if ((f->intr.pbdma.device_fatal_0 |
1600                      f->intr.pbdma.channel_fatal_0 |
1601                      f->intr.pbdma.restartable_0) & pbdma_intr_0) {
1602                         gk20a_err(dev_from_gk20a(g),
1603                                 "pbdma_intr_0(%d):0x%08x PBH: %08x SHADOW: %08x M0: %08x %08x %08x %08x",
1604                                 pbdma_id, pbdma_intr_0,
1605                                 gk20a_readl(g, pbdma_pb_header_r(pbdma_id)),
1606                                 gk20a_readl(g, pbdma_hdr_shadow_r(pbdma_id)),
1607                                 gk20a_readl(g, pbdma_method0_r(pbdma_id)),
1608                                 gk20a_readl(g, pbdma_method1_r(pbdma_id)),
1609                                 gk20a_readl(g, pbdma_method2_r(pbdma_id)),
1610                                 gk20a_readl(g, pbdma_method3_r(pbdma_id))
1611                                 );
1612                         reset = true;
1613                         handled |= ((f->intr.pbdma.device_fatal_0 |
1614                                      f->intr.pbdma.channel_fatal_0 |
1615                                      f->intr.pbdma.restartable_0) &
1616                                     pbdma_intr_0);
1617                 }
1618
1619                 if (pbdma_intr_0 & pbdma_intr_0_pbentry_pending_f()) {
1620                         gk20a_fifo_reset_pbdma_header(g, pbdma_id);
1621                         gk20a_fifo_reset_pbdma_method(g, pbdma_id, 0);
1622                         reset = true;
1623                 }
1624
1625                 if (pbdma_intr_0 & pbdma_intr_0_method_pending_f()) {
1626                         gk20a_fifo_reset_pbdma_method(g, pbdma_id, 0);
1627                         reset = true;
1628                 }
1629
1630                 if (pbdma_intr_0 & pbdma_intr_0_device_pending_f()) {
1631                         gk20a_fifo_reset_pbdma_header(g, pbdma_id);
1632
1633                         for (i = 0; i < 4; i++) {
1634                                 if (gk20a_fifo_is_sw_method_subch(g,
1635                                                 pbdma_id, i))
1636                                         gk20a_fifo_reset_pbdma_method(g,
1637                                                         pbdma_id, i);
1638                         }
1639                         reset = true;
1640                 }
1641
1642                 gk20a_writel(g, pbdma_intr_0_r(pbdma_id), pbdma_intr_0);
1643         }
1644
1645         /* all intrs in _intr_1 are "host copy engine" related,
1646          * which gk20a doesn't have. for now just make them channel fatal. */
1647         if (pbdma_intr_1) {
1648                 dev_err(dev, "channel hce error: pbdma_intr_1(%d): 0x%08x",
1649                         pbdma_id, pbdma_intr_1);
1650                 reset = true;
1651                 gk20a_writel(g, pbdma_intr_1_r(pbdma_id), pbdma_intr_1);
1652         }
1653
1654         if (reset) {
1655                 /* Remove the channel from runlist */
1656                 u32 status = gk20a_readl(g, fifo_pbdma_status_r(pbdma_id));
1657                 u32 id = fifo_pbdma_status_id_v(status);
1658                 if (fifo_pbdma_status_id_type_v(status)
1659                                 == fifo_pbdma_status_id_type_chid_v()) {
1660                         struct channel_gk20a *ch = &f->channel[id];
1661
1662                         if (gk20a_channel_get(ch)) {
1663                                 gk20a_set_error_notifier(ch,
1664                                                 NVGPU_CHANNEL_PBDMA_ERROR);
1665                                 gk20a_fifo_recover_ch(g, id, true);
1666                                 gk20a_channel_put(ch);
1667                         }
1668                 } else if (fifo_pbdma_status_id_type_v(status)
1669                                 == fifo_pbdma_status_id_type_tsgid_v()) {
1670                         struct tsg_gk20a *tsg = &f->tsg[id];
1671                         struct channel_gk20a *ch = NULL;
1672
1673                         mutex_lock(&tsg->ch_list_lock);
1674                         list_for_each_entry(ch, &tsg->ch_list, ch_entry) {
1675                                 if (gk20a_channel_get(ch)) {
1676                                         gk20a_set_error_notifier(ch,
1677                                                 NVGPU_CHANNEL_PBDMA_ERROR);
1678                                         gk20a_channel_put(ch);
1679                                 }
1680                         }
1681                         mutex_unlock(&tsg->ch_list_lock);
1682                         gk20a_fifo_recover_tsg(g, id, true);
1683                 }
1684         }
1685
1686         return handled;
1687 }
1688
1689 static u32 fifo_channel_isr(struct gk20a *g, u32 fifo_intr)
1690 {
1691         gk20a_channel_semaphore_wakeup(g);
1692         return fifo_intr_0_channel_intr_pending_f();
1693 }
1694
1695
1696 static u32 fifo_pbdma_isr(struct gk20a *g, u32 fifo_intr)
1697 {
1698         struct device *dev = dev_from_gk20a(g);
1699         struct fifo_gk20a *f = &g->fifo;
1700         u32 clear_intr = 0, i;
1701         u32 pbdma_pending = gk20a_readl(g, fifo_intr_pbdma_id_r());
1702
1703         for (i = 0; i < fifo_intr_pbdma_id_status__size_1_v(); i++) {
1704                 if (fifo_intr_pbdma_id_status_f(pbdma_pending, i)) {
1705                         gk20a_dbg(gpu_dbg_intr, "pbdma id %d intr pending", i);
1706                         clear_intr |=
1707                                 gk20a_fifo_handle_pbdma_intr(dev, g, f, i);
1708                 }
1709         }
1710         return fifo_intr_0_pbdma_intr_pending_f();
1711 }
1712
1713 void gk20a_fifo_isr(struct gk20a *g)
1714 {
1715         u32 error_intr_mask =
1716                 fifo_intr_0_bind_error_pending_f() |
1717                 fifo_intr_0_sched_error_pending_f() |
1718                 fifo_intr_0_chsw_error_pending_f() |
1719                 fifo_intr_0_fb_flush_timeout_pending_f() |
1720                 fifo_intr_0_dropped_mmu_fault_pending_f() |
1721                 fifo_intr_0_mmu_fault_pending_f() |
1722                 fifo_intr_0_lb_error_pending_f() |
1723                 fifo_intr_0_pio_error_pending_f();
1724
1725         u32 fifo_intr = gk20a_readl(g, fifo_intr_0_r());
1726         u32 clear_intr = 0;
1727
1728         if (g->fifo.sw_ready) {
1729                 /* note we're not actually in an "isr", but rather
1730                  * in a threaded interrupt context... */
1731                 mutex_lock(&g->fifo.intr.isr.mutex);
1732
1733                 gk20a_dbg(gpu_dbg_intr, "fifo isr %08x\n", fifo_intr);
1734
1735                 /* handle runlist update */
1736                 if (fifo_intr & fifo_intr_0_runlist_event_pending_f()) {
1737                         gk20a_fifo_handle_runlist_event(g);
1738                         clear_intr |= fifo_intr_0_runlist_event_pending_f();
1739                 }
1740                 if (fifo_intr & fifo_intr_0_pbdma_intr_pending_f())
1741                         clear_intr |= fifo_pbdma_isr(g, fifo_intr);
1742
1743                 if (unlikely(fifo_intr & error_intr_mask))
1744                         clear_intr = fifo_error_isr(g, fifo_intr);
1745
1746                 mutex_unlock(&g->fifo.intr.isr.mutex);
1747         }
1748         gk20a_writel(g, fifo_intr_0_r(), clear_intr);
1749
1750         return;
1751 }
1752
1753 void gk20a_fifo_nonstall_isr(struct gk20a *g)
1754 {
1755         u32 fifo_intr = gk20a_readl(g, fifo_intr_0_r());
1756         u32 clear_intr = 0;
1757
1758         gk20a_dbg(gpu_dbg_intr, "fifo nonstall isr %08x\n", fifo_intr);
1759
1760         if (fifo_intr & fifo_intr_0_channel_intr_pending_f())
1761                 clear_intr |= fifo_channel_isr(g, fifo_intr);
1762
1763         gk20a_writel(g, fifo_intr_0_r(), clear_intr);
1764
1765         return;
1766 }
1767
1768 static int __locked_fifo_preempt(struct gk20a *g, u32 id, bool is_tsg)
1769 {
1770         u32 delay = GR_IDLE_CHECK_DEFAULT;
1771         unsigned long end_jiffies = jiffies
1772                 + msecs_to_jiffies(gk20a_get_gr_idle_timeout(g));
1773         u32 ret = 0;
1774
1775         gk20a_dbg_fn("%d", id);
1776
1777         /* issue preempt */
1778         if (is_tsg)
1779                 gk20a_writel(g, fifo_preempt_r(),
1780                         fifo_preempt_id_f(id) |
1781                         fifo_preempt_type_tsg_f());
1782         else
1783                 gk20a_writel(g, fifo_preempt_r(),
1784                         fifo_preempt_chid_f(id) |
1785                         fifo_preempt_type_channel_f());
1786
1787         gk20a_dbg_fn("%d", id);
1788         /* wait for preempt */
1789         ret = -EBUSY;
1790         do {
1791                 if (!(gk20a_readl(g, fifo_preempt_r()) &
1792                         fifo_preempt_pending_true_f())) {
1793                         ret = 0;
1794                         break;
1795                 }
1796
1797                 usleep_range(delay, delay * 2);
1798                 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
1799         } while (time_before(jiffies, end_jiffies) ||
1800                         !tegra_platform_is_silicon());
1801
1802         gk20a_dbg_fn("%d", id);
1803         if (ret) {
1804                 if (is_tsg) {
1805                         struct tsg_gk20a *tsg = &g->fifo.tsg[id];
1806                         struct channel_gk20a *ch = NULL;
1807
1808                         gk20a_err(dev_from_gk20a(g),
1809                                 "preempt TSG %d timeout\n", id);
1810
1811                         mutex_lock(&tsg->ch_list_lock);
1812                         list_for_each_entry(ch, &tsg->ch_list, ch_entry) {
1813                                 if (!gk20a_channel_get(ch))
1814                                         continue;
1815                                 gk20a_set_error_notifier(ch,
1816                                         NVGPU_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT);
1817                                 gk20a_channel_put(ch);
1818                         }
1819                         mutex_unlock(&tsg->ch_list_lock);
1820                         gk20a_fifo_recover_tsg(g, id, true);
1821                 } else {
1822                         struct channel_gk20a *ch = &g->fifo.channel[id];
1823
1824                         gk20a_err(dev_from_gk20a(g),
1825                                 "preempt channel %d timeout\n", id);
1826
1827                         if (gk20a_channel_get(ch)) {
1828                                 gk20a_set_error_notifier(ch,
1829                                                 NVGPU_CHANNEL_FIFO_ERROR_IDLE_TIMEOUT);
1830                                 gk20a_fifo_recover_ch(g, id, true);
1831                                 gk20a_channel_put(ch);
1832                         }
1833                 }
1834         }
1835
1836         return ret;
1837 }
1838
1839 int gk20a_fifo_preempt_channel(struct gk20a *g, u32 hw_chid)
1840 {
1841         struct fifo_gk20a *f = &g->fifo;
1842         u32 ret = 0;
1843         u32 token = PMU_INVALID_MUTEX_OWNER_ID;
1844         u32 mutex_ret = 0;
1845         u32 i;
1846
1847         gk20a_dbg_fn("%d", hw_chid);
1848
1849         /* we have no idea which runlist we are using. lock all */
1850         for (i = 0; i < g->fifo.max_runlists; i++)
1851                 mutex_lock(&f->runlist_info[i].mutex);
1852
1853         mutex_ret = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1854
1855         ret = __locked_fifo_preempt(g, hw_chid, false);
1856
1857         if (!mutex_ret)
1858                 pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1859
1860         for (i = 0; i < g->fifo.max_runlists; i++)
1861                 mutex_unlock(&f->runlist_info[i].mutex);
1862
1863         return ret;
1864 }
1865
1866 int gk20a_fifo_preempt_tsg(struct gk20a *g, u32 tsgid)
1867 {
1868         struct fifo_gk20a *f = &g->fifo;
1869         u32 ret = 0;
1870         u32 token = PMU_INVALID_MUTEX_OWNER_ID;
1871         u32 mutex_ret = 0;
1872         u32 i;
1873
1874         gk20a_dbg_fn("%d", tsgid);
1875
1876         /* we have no idea which runlist we are using. lock all */
1877         for (i = 0; i < g->fifo.max_runlists; i++)
1878                 mutex_lock(&f->runlist_info[i].mutex);
1879
1880         mutex_ret = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1881
1882         ret = __locked_fifo_preempt(g, tsgid, true);
1883
1884         if (!mutex_ret)
1885                 pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1886
1887         for (i = 0; i < g->fifo.max_runlists; i++)
1888                 mutex_unlock(&f->runlist_info[i].mutex);
1889
1890         return ret;
1891 }
1892
1893 int gk20a_fifo_preempt(struct gk20a *g, struct channel_gk20a *ch)
1894 {
1895         int err;
1896
1897         if (gk20a_is_channel_marked_as_tsg(ch))
1898                 err = gk20a_fifo_preempt_tsg(ch->g, ch->tsgid);
1899         else
1900                 err = gk20a_fifo_preempt_channel(ch->g, ch->hw_chid);
1901
1902         return err;
1903 }
1904
1905 int gk20a_fifo_enable_engine_activity(struct gk20a *g,
1906                                 struct fifo_engine_info_gk20a *eng_info)
1907 {
1908         u32 token = PMU_INVALID_MUTEX_OWNER_ID;
1909         u32 mutex_ret;
1910         u32 enable;
1911
1912         gk20a_dbg_fn("");
1913
1914         mutex_ret = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1915
1916         enable = gk20a_readl(g, fifo_sched_disable_r());
1917         enable &= ~(fifo_sched_disable_true_v() >> eng_info->runlist_id);
1918         gk20a_writel(g, fifo_sched_disable_r(), enable);
1919
1920         if (!mutex_ret)
1921                 pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1922
1923         gk20a_dbg_fn("done");
1924         return 0;
1925 }
1926
1927 int gk20a_fifo_disable_engine_activity(struct gk20a *g,
1928                                 struct fifo_engine_info_gk20a *eng_info,
1929                                 bool wait_for_idle)
1930 {
1931         u32 gr_stat, pbdma_stat, chan_stat, eng_stat, ctx_stat;
1932         u32 pbdma_chid = ~0, engine_chid = ~0, disable;
1933         u32 token = PMU_INVALID_MUTEX_OWNER_ID;
1934         u32 mutex_ret;
1935         u32 err = 0;
1936
1937         gk20a_dbg_fn("");
1938
1939         gr_stat =
1940                 gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id));
1941         if (fifo_engine_status_engine_v(gr_stat) ==
1942             fifo_engine_status_engine_busy_v() && !wait_for_idle)
1943                 return -EBUSY;
1944
1945         mutex_ret = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1946
1947         disable = gk20a_readl(g, fifo_sched_disable_r());
1948         disable = set_field(disable,
1949                         fifo_sched_disable_runlist_m(eng_info->runlist_id),
1950                         fifo_sched_disable_runlist_f(fifo_sched_disable_true_v(),
1951                                 eng_info->runlist_id));
1952         gk20a_writel(g, fifo_sched_disable_r(), disable);
1953
1954         /* chid from pbdma status */
1955         pbdma_stat = gk20a_readl(g, fifo_pbdma_status_r(eng_info->pbdma_id));
1956         chan_stat  = fifo_pbdma_status_chan_status_v(pbdma_stat);
1957         if (chan_stat == fifo_pbdma_status_chan_status_valid_v() ||
1958             chan_stat == fifo_pbdma_status_chan_status_chsw_save_v())
1959                 pbdma_chid = fifo_pbdma_status_id_v(pbdma_stat);
1960         else if (chan_stat == fifo_pbdma_status_chan_status_chsw_load_v() ||
1961                  chan_stat == fifo_pbdma_status_chan_status_chsw_switch_v())
1962                 pbdma_chid = fifo_pbdma_status_next_id_v(pbdma_stat);
1963
1964         if (pbdma_chid != ~0) {
1965                 err = g->ops.fifo.preempt_channel(g, pbdma_chid);
1966                 if (err)
1967                         goto clean_up;
1968         }
1969
1970         /* chid from engine status */
1971         eng_stat = gk20a_readl(g, fifo_engine_status_r(eng_info->engine_id));
1972         ctx_stat  = fifo_engine_status_ctx_status_v(eng_stat);
1973         if (ctx_stat == fifo_engine_status_ctx_status_valid_v() ||
1974             ctx_stat == fifo_engine_status_ctx_status_ctxsw_save_v())
1975                 engine_chid = fifo_engine_status_id_v(eng_stat);
1976         else if (ctx_stat == fifo_engine_status_ctx_status_ctxsw_load_v() ||
1977                  ctx_stat == fifo_engine_status_ctx_status_ctxsw_switch_v())
1978                 engine_chid = fifo_engine_status_next_id_v(eng_stat);
1979
1980         if (engine_chid != ~0 && engine_chid != pbdma_chid) {
1981                 err = g->ops.fifo.preempt_channel(g, engine_chid);
1982                 if (err)
1983                         goto clean_up;
1984         }
1985
1986 clean_up:
1987         if (!mutex_ret)
1988                 pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
1989
1990         if (err) {
1991                 gk20a_dbg_fn("failed");
1992                 if (gk20a_fifo_enable_engine_activity(g, eng_info))
1993                         gk20a_err(dev_from_gk20a(g),
1994                                 "failed to enable gr engine activity\n");
1995         } else {
1996                 gk20a_dbg_fn("done");
1997         }
1998         return err;
1999 }
2000
2001 static void gk20a_fifo_runlist_reset_engines(struct gk20a *g, u32 runlist_id)
2002 {
2003         struct fifo_gk20a *f = &g->fifo;
2004         u32 engines = 0;
2005         int i;
2006
2007         for (i = 0; i < f->max_engines; i++) {
2008                 u32 status = gk20a_readl(g, fifo_engine_status_r(i));
2009                 bool engine_busy = fifo_engine_status_engine_v(status) ==
2010                         fifo_engine_status_engine_busy_v();
2011
2012                 if (engine_busy &&
2013                     (f->engine_info[i].runlist_id == runlist_id))
2014                         engines |= BIT(i);
2015         }
2016
2017         if (engines)
2018                 gk20a_fifo_recover(g, engines, ~(u32)0, false, false, true);
2019 }
2020
2021 static int gk20a_fifo_runlist_wait_pending(struct gk20a *g, u32 runlist_id)
2022 {
2023         struct fifo_runlist_info_gk20a *runlist;
2024         unsigned long end_jiffies = jiffies +
2025                 msecs_to_jiffies(gk20a_get_gr_idle_timeout(g));
2026         unsigned long delay = GR_IDLE_CHECK_DEFAULT;
2027         int ret = -ETIMEDOUT;
2028
2029         runlist = &g->fifo.runlist_info[runlist_id];
2030         do {
2031                 if ((gk20a_readl(g, fifo_eng_runlist_r(runlist_id)) &
2032                                 fifo_eng_runlist_pending_true_f()) == 0) {
2033                         ret = 0;
2034                         break;
2035                 }
2036
2037                 usleep_range(delay, delay * 2);
2038                 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
2039         } while (time_before(jiffies, end_jiffies) ||
2040                  !tegra_platform_is_silicon());
2041
2042         return ret;
2043 }
2044
2045 static int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
2046                                             u32 hw_chid, bool add,
2047                                             bool wait_for_finish)
2048 {
2049         u32 ret = 0;
2050         struct fifo_gk20a *f = &g->fifo;
2051         struct fifo_runlist_info_gk20a *runlist = NULL;
2052         u32 *runlist_entry_base = NULL;
2053         u32 *runlist_entry = NULL;
2054         phys_addr_t runlist_pa;
2055         u32 old_buf, new_buf;
2056         u32 chid, tsgid;
2057         struct channel_gk20a *ch = NULL;
2058         struct tsg_gk20a *tsg = NULL;
2059         u32 count = 0;
2060         u32 count_channels_in_tsg;
2061         runlist = &f->runlist_info[runlist_id];
2062
2063         /* valid channel, add/remove it from active list.
2064            Otherwise, keep active list untouched for suspend/resume. */
2065         if (hw_chid != ~0) {
2066                 ch = &f->channel[hw_chid];
2067                 if (gk20a_is_channel_marked_as_tsg(ch))
2068                         tsg = &f->tsg[ch->tsgid];
2069
2070                 if (add) {
2071                         if (test_and_set_bit(hw_chid,
2072                                 runlist->active_channels) == 1)
2073                                 return 0;
2074                         if (tsg && ++tsg->num_active_channels)
2075                                 set_bit(f->channel[hw_chid].tsgid,
2076                                         runlist->active_tsgs);
2077                 } else {
2078                         if (test_and_clear_bit(hw_chid,
2079                                 runlist->active_channels) == 0)
2080                                 return 0;
2081                         if (tsg && --tsg->num_active_channels == 0)
2082                                 clear_bit(f->channel[hw_chid].tsgid,
2083                                         runlist->active_tsgs);
2084                 }
2085         }
2086
2087         old_buf = runlist->cur_buffer;
2088         new_buf = !runlist->cur_buffer;
2089
2090         gk20a_dbg_info("runlist_id : %d, switch to new buffer 0x%16llx",
2091                 runlist_id, (u64)gk20a_mem_phys(&runlist->mem[new_buf]));
2092
2093         runlist_pa = gk20a_mem_phys(&runlist->mem[new_buf]);
2094         if (!runlist_pa) {
2095                 ret = -EINVAL;
2096                 goto clean_up;
2097         }
2098
2099         runlist_entry_base = runlist->mem[new_buf].cpu_va;
2100         if (!runlist_entry_base) {
2101                 ret = -ENOMEM;
2102                 goto clean_up;
2103         }
2104
2105         if (hw_chid != ~0 || /* add/remove a valid channel */
2106             add /* resume to add all channels back */) {
2107                 runlist_entry = runlist_entry_base;
2108
2109                 /* add non-TSG channels first */
2110                 for_each_set_bit(chid,
2111                         runlist->active_channels, f->num_channels) {
2112                         ch = &f->channel[chid];
2113
2114                         if (!gk20a_is_channel_marked_as_tsg(ch)) {
2115                                 gk20a_dbg_info("add channel %d to runlist",
2116                                         chid);
2117                                 runlist_entry[0] = ram_rl_entry_chid_f(chid);
2118                                 runlist_entry[1] = 0;
2119                                 runlist_entry += 2;
2120                                 count++;
2121                         }
2122                 }
2123
2124                 /* now add TSG entries and channels bound to TSG */
2125                 mutex_lock(&f->tsg_inuse_mutex);
2126                 for_each_set_bit(tsgid,
2127                                 runlist->active_tsgs, f->num_channels) {
2128                         tsg = &f->tsg[tsgid];
2129                         /* add TSG entry */
2130                         gk20a_dbg_info("add TSG %d to runlist", tsg->tsgid);
2131                         runlist_entry[0] = ram_rl_entry_id_f(tsg->tsgid) |
2132                                 ram_rl_entry_type_tsg_f() |
2133                                 ram_rl_entry_timeslice_scale_3_f() |
2134                                 ram_rl_entry_timeslice_timeout_128_f() |
2135                                 ram_rl_entry_tsg_length_f(
2136                                         tsg->num_active_channels);
2137                         runlist_entry[1] = 0;
2138                         runlist_entry += 2;
2139                         count++;
2140
2141                         /* add runnable channels bound to this TSG */
2142                         count_channels_in_tsg = 0;
2143                         mutex_lock(&tsg->ch_list_lock);
2144                         list_for_each_entry(ch, &tsg->ch_list, ch_entry) {
2145                                 if (!test_bit(ch->hw_chid,
2146                                                 runlist->active_channels))
2147                                         continue;
2148                                 gk20a_dbg_info("add channel %d to runlist",
2149                                         ch->hw_chid);
2150                                 runlist_entry[0] =
2151                                         ram_rl_entry_chid_f(ch->hw_chid);
2152                                 runlist_entry[1] = 0;
2153                                 runlist_entry += 2;
2154                                 count++;
2155                                 count_channels_in_tsg++;
2156                         }
2157                         mutex_unlock(&tsg->ch_list_lock);
2158
2159                         WARN_ON(tsg->num_active_channels !=
2160                                 count_channels_in_tsg);
2161                 }
2162                 mutex_unlock(&f->tsg_inuse_mutex);
2163         } else  /* suspend to remove all channels */
2164                 count = 0;
2165
2166         if (count != 0) {
2167                 gk20a_writel(g, fifo_runlist_base_r(),
2168                         fifo_runlist_base_ptr_f(u64_lo32(runlist_pa >> 12)) |
2169                         fifo_runlist_base_target_vid_mem_f());
2170         }
2171
2172         gk20a_writel(g, fifo_runlist_r(),
2173                 fifo_runlist_engine_f(runlist_id) |
2174                 fifo_eng_runlist_length_f(count));
2175
2176         if (wait_for_finish) {
2177                 ret = gk20a_fifo_runlist_wait_pending(g, runlist_id);
2178
2179                 if (ret == -ETIMEDOUT) {
2180                         gk20a_err(dev_from_gk20a(g),
2181                                    "runlist update timeout");
2182
2183                         gk20a_fifo_runlist_reset_engines(g, runlist_id);
2184
2185                         /* engine reset needs the lock. drop it */
2186                         /* wait until the runlist is active again */
2187                         ret = gk20a_fifo_runlist_wait_pending(g, runlist_id);
2188                         /* get the lock back. at this point everything should
2189                          * should be fine */
2190
2191                         if (ret)
2192                                 gk20a_err(dev_from_gk20a(g),
2193                                            "runlist update failed: %d", ret);
2194                 } else if (ret == -EINTR)
2195                         gk20a_err(dev_from_gk20a(g),
2196                                    "runlist update interrupted");
2197         }
2198
2199         runlist->cur_buffer = new_buf;
2200
2201 clean_up:
2202         return ret;
2203 }
2204
2205 /* add/remove a channel from runlist
2206    special cases below: runlist->active_channels will NOT be changed.
2207    (hw_chid == ~0 && !add) means remove all active channels from runlist.
2208    (hw_chid == ~0 &&  add) means restore all active channels on runlist. */
2209 int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 hw_chid,
2210                               bool add, bool wait_for_finish)
2211 {
2212         struct fifo_runlist_info_gk20a *runlist = NULL;
2213         struct fifo_gk20a *f = &g->fifo;
2214         u32 token = PMU_INVALID_MUTEX_OWNER_ID;
2215         u32 mutex_ret;
2216         u32 ret = 0;
2217
2218         gk20a_dbg_fn("");
2219
2220         runlist = &f->runlist_info[runlist_id];
2221
2222         mutex_lock(&runlist->mutex);
2223
2224         mutex_ret = pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
2225
2226         ret = gk20a_fifo_update_runlist_locked(g, runlist_id, hw_chid, add,
2227                                                wait_for_finish);
2228
2229         if (!mutex_ret)
2230                 pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
2231
2232         mutex_unlock(&runlist->mutex);
2233         return ret;
2234 }
2235
2236 int gk20a_fifo_suspend(struct gk20a *g)
2237 {
2238         gk20a_dbg_fn("");
2239
2240         /* stop bar1 snooping */
2241         gk20a_writel(g, fifo_bar1_base_r(),
2242                         fifo_bar1_base_valid_false_f());
2243
2244         /* disable fifo intr */
2245         gk20a_writel(g, fifo_intr_en_0_r(), 0);
2246         gk20a_writel(g, fifo_intr_en_1_r(), 0);
2247
2248         gk20a_dbg_fn("done");
2249         return 0;
2250 }
2251
2252 bool gk20a_fifo_mmu_fault_pending(struct gk20a *g)
2253 {
2254         if (gk20a_readl(g, fifo_intr_0_r()) &
2255                         fifo_intr_0_mmu_fault_pending_f())
2256                 return true;
2257         else
2258                 return false;
2259 }
2260
2261 int gk20a_fifo_wait_engine_idle(struct gk20a *g)
2262 {
2263         unsigned long end_jiffies = jiffies +
2264                 msecs_to_jiffies(gk20a_get_gr_idle_timeout(g));
2265         unsigned long delay = GR_IDLE_CHECK_DEFAULT;
2266         int ret = -ETIMEDOUT;
2267         u32 i;
2268         struct device *d = dev_from_gk20a(g);
2269
2270         gk20a_dbg_fn("");
2271
2272         for (i = 0; i < fifo_engine_status__size_1_v(); i++) {
2273                 do {
2274                         u32 status = gk20a_readl(g, fifo_engine_status_r(i));
2275                         if (!fifo_engine_status_engine_v(status)) {
2276                                 ret = 0;
2277                                 break;
2278                         }
2279
2280                         usleep_range(delay, delay * 2);
2281                         delay = min_t(unsigned long,
2282                                         delay << 1, GR_IDLE_CHECK_MAX);
2283                 } while (time_before(jiffies, end_jiffies) ||
2284                                 !tegra_platform_is_silicon());
2285                 if (ret) {
2286                         gk20a_err(d, "cannot idle engine %u\n", i);
2287                         break;
2288                 }
2289         }
2290
2291         gk20a_dbg_fn("done");
2292
2293         return ret;
2294 }
2295
2296 static void gk20a_fifo_apply_pb_timeout(struct gk20a *g)
2297 {
2298         u32 timeout;
2299
2300         if (tegra_platform_is_silicon()) {
2301                 timeout = gk20a_readl(g, fifo_pb_timeout_r());
2302                 timeout &= ~fifo_pb_timeout_detection_enabled_f();
2303                 gk20a_writel(g, fifo_pb_timeout_r(), timeout);
2304         }
2305 }
2306
2307 static u32 gk20a_fifo_get_num_fifos(struct gk20a *g)
2308 {
2309         return ccsr_channel__size_1_v();
2310 }
2311
2312 u32 gk20a_fifo_get_pbdma_signature(struct gk20a *g)
2313 {
2314         return pbdma_signature_hw_valid_f() | pbdma_signature_sw_zero_f();
2315 }
2316
2317 void gk20a_init_fifo(struct gpu_ops *gops)
2318 {
2319         gk20a_init_channel(gops);
2320         gops->fifo.preempt_channel = gk20a_fifo_preempt_channel;
2321         gops->fifo.update_runlist = gk20a_fifo_update_runlist;
2322         gops->fifo.trigger_mmu_fault = gk20a_fifo_trigger_mmu_fault;
2323         gops->fifo.apply_pb_timeout = gk20a_fifo_apply_pb_timeout;
2324         gops->fifo.wait_engine_idle = gk20a_fifo_wait_engine_idle;
2325         gops->fifo.get_num_fifos = gk20a_fifo_get_num_fifos;
2326         gops->fifo.get_pbdma_signature = gk20a_fifo_get_pbdma_signature;
2327 }