drivers/gpu/nvgpu/gk20a/channel_gk20a.c

   1 /*
   2  * drivers/video/tegra/host/gk20a/channel_gk20a.c
   3  *
   4  * GK20A Graphics channel
   5  *
   6  * Copyright (c) 2011-2014, NVIDIA CORPORATION.  All rights reserved.
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms and conditions of the GNU General Public License,
  10  * version 2, as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope it will be useful, but WITHOUT
  13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15  * more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along with
  18  * this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  20  */
  21
  22 #include <linux/nvhost.h>
  23 #include <linux/list.h>
  24 #include <linux/delay.h>
  25 #include <linux/highmem.h> /* need for nvmap.h*/
  26 #include <trace/events/gk20a.h>
  27 #include <linux/scatterlist.h>
  28 #include <linux/file.h>
  29 #include <linux/anon_inodes.h>
  30 #include <linux/dma-buf.h>
  31
  32 #include "debug_gk20a.h"
  33
  34 #include "gk20a.h"
  35 #include "dbg_gpu_gk20a.h"
  36
  37 #include "hw_ram_gk20a.h"
  38 #include "hw_fifo_gk20a.h"
  39 #include "hw_pbdma_gk20a.h"
  40 #include "hw_ccsr_gk20a.h"
  41 #include "hw_ltc_gk20a.h"
  42
  43 #define NVMAP_HANDLE_PARAM_SIZE 1
  44
  45 static struct channel_gk20a *acquire_unused_channel(struct fifo_gk20a *f);
  46 static void release_used_channel(struct fifo_gk20a *f, struct channel_gk20a *c);
  47
  48 static void free_priv_cmdbuf(struct channel_gk20a *c,
  49                              struct priv_cmd_entry *e);
  50 static void recycle_priv_cmdbuf(struct channel_gk20a *c);
  51
  52 static int channel_gk20a_alloc_priv_cmdbuf(struct channel_gk20a *c);
  53 static void channel_gk20a_free_priv_cmdbuf(struct channel_gk20a *c);
  54
  55 static int channel_gk20a_commit_userd(struct channel_gk20a *c);
  56 static int channel_gk20a_setup_userd(struct channel_gk20a *c);
  57 static int channel_gk20a_setup_ramfc(struct channel_gk20a *c,
  58                         u64 gpfifo_base, u32 gpfifo_entries);
  59
  60 static void channel_gk20a_bind(struct channel_gk20a *ch_gk20a);
  61 static void channel_gk20a_unbind(struct channel_gk20a *ch_gk20a);
  62
  63 static int channel_gk20a_alloc_inst(struct gk20a *g,
  64                                 struct channel_gk20a *ch);
  65 static void channel_gk20a_free_inst(struct gk20a *g,
  66                                 struct channel_gk20a *ch);
  67
  68 static int channel_gk20a_update_runlist(struct channel_gk20a *c,
  69                                         bool add);
  70 static void gk20a_free_error_notifiers(struct channel_gk20a *ch);
  71
  72 static struct channel_gk20a *acquire_unused_channel(struct fifo_gk20a *f)
  73 {
  74         struct channel_gk20a *ch = NULL;
  75         int chid;
  76
  77         mutex_lock(&f->ch_inuse_mutex);
  78         for (chid = 0; chid < f->num_channels; chid++) {
  79                 if (!f->channel[chid].in_use) {
  80                         f->channel[chid].in_use = true;
  81                         ch = &f->channel[chid];
  82                         break;
  83                 }
  84         }
  85         mutex_unlock(&f->ch_inuse_mutex);
  86
  87         return ch;
  88 }
  89
  90 static void release_used_channel(struct fifo_gk20a *f, struct channel_gk20a *c)
  91 {
  92         mutex_lock(&f->ch_inuse_mutex);
  93         f->channel[c->hw_chid].in_use = false;
  94         mutex_unlock(&f->ch_inuse_mutex);
  95 }
  96
  97 int channel_gk20a_commit_va(struct channel_gk20a *c)
  98 {
  99         u64 addr;
 100         u32 addr_lo;
 101         u32 addr_hi;
 102         void *inst_ptr;
 103
 104         gk20a_dbg_fn("");
 105
 106         inst_ptr = c->inst_block.cpuva;
 107         if (!inst_ptr)
 108                 return -ENOMEM;
 109
 110         addr = gk20a_mm_iova_addr(c->vm->pdes.sgt->sgl);
 111         addr_lo = u64_lo32(addr >> 12);
 112         addr_hi = u64_hi32(addr);
 113
 114         gk20a_dbg_info("pde pa=0x%llx addr_lo=0x%x addr_hi=0x%x",
 115                    (u64)addr, addr_lo, addr_hi);
 116
 117         gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_lo_w(),
 118                 ram_in_page_dir_base_target_vid_mem_f() |
 119                 ram_in_page_dir_base_vol_true_f() |
 120                 ram_in_page_dir_base_lo_f(addr_lo));
 121
 122         gk20a_mem_wr32(inst_ptr, ram_in_page_dir_base_hi_w(),
 123                 ram_in_page_dir_base_hi_f(addr_hi));
 124
 125         gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_lo_w(),
 126                  u64_lo32(c->vm->va_limit) | 0xFFF);
 127
 128         gk20a_mem_wr32(inst_ptr, ram_in_adr_limit_hi_w(),
 129                 ram_in_adr_limit_hi_f(u64_hi32(c->vm->va_limit)));
 130
 131         return 0;
 132 }
 133
 134 static int channel_gk20a_commit_userd(struct channel_gk20a *c)
 135 {
 136         u32 addr_lo;
 137         u32 addr_hi;
 138         void *inst_ptr;
 139
 140         gk20a_dbg_fn("");
 141
 142         inst_ptr = c->inst_block.cpuva;
 143         if (!inst_ptr)
 144                 return -ENOMEM;
 145
 146         addr_lo = u64_lo32(c->userd_iova >> ram_userd_base_shift_v());
 147         addr_hi = u64_hi32(c->userd_iova);
 148
 149         gk20a_dbg_info("channel %d : set ramfc userd 0x%16llx",
 150                 c->hw_chid, (u64)c->userd_iova);
 151
 152         gk20a_mem_wr32(inst_ptr, ram_in_ramfc_w() + ram_fc_userd_w(),
 153                  pbdma_userd_target_vid_mem_f() |
 154                  pbdma_userd_addr_f(addr_lo));
 155
 156         gk20a_mem_wr32(inst_ptr, ram_in_ramfc_w() + ram_fc_userd_hi_w(),
 157                  pbdma_userd_target_vid_mem_f() |
 158                  pbdma_userd_hi_addr_f(addr_hi));
 159
 160         return 0;
 161 }
 162
 163 static int channel_gk20a_set_schedule_params(struct channel_gk20a *c,
 164                                 u32 timeslice_timeout)
 165 {
 166         void *inst_ptr;
 167         int shift = 3;
 168         int value = timeslice_timeout;
 169
 170         inst_ptr = c->inst_block.cpuva;
 171         if (!inst_ptr)
 172                 return -ENOMEM;
 173
 174         /* disable channel */
 175         gk20a_writel(c->g, ccsr_channel_r(c->hw_chid),
 176                 gk20a_readl(c->g, ccsr_channel_r(c->hw_chid)) |
 177                 ccsr_channel_enable_clr_true_f());
 178
 179         /* preempt the channel */
 180         WARN_ON(gk20a_fifo_preempt_channel(c->g, c->hw_chid));
 181
 182         /* value field is 8 bits long */
 183         while (value >= 1 << 8) {
 184                 value >>= 1;
 185                 shift++;
 186         }
 187
 188         /* time slice register is only 18bits long */
 189         if ((value << shift) >= 1<<19) {
 190                 pr_err("Requested timeslice value is clamped to 18 bits\n");
 191                 value = 255;
 192                 shift = 10;
 193         }
 194
 195         /* set new timeslice */
 196         gk20a_mem_wr32(inst_ptr, ram_fc_eng_timeslice_w(),
 197                 value | (shift << 12) |
 198                 fifo_eng_timeslice_enable_true_f());
 199
 200         /* enable channel */
 201         gk20a_writel(c->g, ccsr_channel_r(c->hw_chid),
 202                 gk20a_readl(c->g, ccsr_channel_r(c->hw_chid)) |
 203                 ccsr_channel_enable_set_true_f());
 204
 205         return 0;
 206 }
 207
 208 static int channel_gk20a_setup_ramfc(struct channel_gk20a *c,
 209                                 u64 gpfifo_base, u32 gpfifo_entries)
 210 {
 211         void *inst_ptr;
 212
 213         gk20a_dbg_fn("");
 214
 215         inst_ptr = c->inst_block.cpuva;
 216         if (!inst_ptr)
 217                 return -ENOMEM;
 218
 219         memset(inst_ptr, 0, ram_fc_size_val_v());
 220
 221         gk20a_mem_wr32(inst_ptr, ram_fc_gp_base_w(),
 222                 pbdma_gp_base_offset_f(
 223                 u64_lo32(gpfifo_base >> pbdma_gp_base_rsvd_s())));
 224
 225         gk20a_mem_wr32(inst_ptr, ram_fc_gp_base_hi_w(),
 226                 pbdma_gp_base_hi_offset_f(u64_hi32(gpfifo_base)) |
 227                 pbdma_gp_base_hi_limit2_f(ilog2(gpfifo_entries)));
 228
 229         gk20a_mem_wr32(inst_ptr, ram_fc_signature_w(),
 230                  pbdma_signature_hw_valid_f() | pbdma_signature_sw_zero_f());
 231
 232         gk20a_mem_wr32(inst_ptr, ram_fc_formats_w(),
 233                 pbdma_formats_gp_fermi0_f() |
 234                 pbdma_formats_pb_fermi1_f() |
 235                 pbdma_formats_mp_fermi0_f());
 236
 237         gk20a_mem_wr32(inst_ptr, ram_fc_pb_header_w(),
 238                 pbdma_pb_header_priv_user_f() |
 239                 pbdma_pb_header_method_zero_f() |
 240                 pbdma_pb_header_subchannel_zero_f() |
 241                 pbdma_pb_header_level_main_f() |
 242                 pbdma_pb_header_first_true_f() |
 243                 pbdma_pb_header_type_inc_f());
 244
 245         gk20a_mem_wr32(inst_ptr, ram_fc_subdevice_w(),
 246                 pbdma_subdevice_id_f(1) |
 247                 pbdma_subdevice_status_active_f() |
 248                 pbdma_subdevice_channel_dma_enable_f());
 249
 250         gk20a_mem_wr32(inst_ptr, ram_fc_target_w(), pbdma_target_engine_sw_f());
 251
 252         gk20a_mem_wr32(inst_ptr, ram_fc_acquire_w(),
 253                 pbdma_acquire_retry_man_2_f() |
 254                 pbdma_acquire_retry_exp_2_f() |
 255                 pbdma_acquire_timeout_exp_max_f() |
 256                 pbdma_acquire_timeout_man_max_f() |
 257                 pbdma_acquire_timeout_en_disable_f());
 258
 259         gk20a_mem_wr32(inst_ptr, ram_fc_eng_timeslice_w(),
 260                 fifo_eng_timeslice_timeout_128_f() |
 261                 fifo_eng_timeslice_timescale_3_f() |
 262                 fifo_eng_timeslice_enable_true_f());
 263
 264         gk20a_mem_wr32(inst_ptr, ram_fc_pb_timeslice_w(),
 265                 fifo_pb_timeslice_timeout_16_f() |
 266                 fifo_pb_timeslice_timescale_0_f() |
 267                 fifo_pb_timeslice_enable_true_f());
 268
 269         gk20a_mem_wr32(inst_ptr, ram_fc_chid_w(), ram_fc_chid_id_f(c->hw_chid));
 270
 271         return 0;
 272 }
 273
 274 static int channel_gk20a_setup_userd(struct channel_gk20a *c)
 275 {
 276         BUG_ON(!c->userd_cpu_va);
 277
 278         gk20a_dbg_fn("");
 279
 280         gk20a_mem_wr32(c->userd_cpu_va, ram_userd_put_w(), 0);
 281         gk20a_mem_wr32(c->userd_cpu_va, ram_userd_get_w(), 0);
 282         gk20a_mem_wr32(c->userd_cpu_va, ram_userd_ref_w(), 0);
 283         gk20a_mem_wr32(c->userd_cpu_va, ram_userd_put_hi_w(), 0);
 284         gk20a_mem_wr32(c->userd_cpu_va, ram_userd_ref_threshold_w(), 0);
 285         gk20a_mem_wr32(c->userd_cpu_va, ram_userd_gp_top_level_get_w(), 0);
 286         gk20a_mem_wr32(c->userd_cpu_va, ram_userd_gp_top_level_get_hi_w(), 0);
 287         gk20a_mem_wr32(c->userd_cpu_va, ram_userd_get_hi_w(), 0);
 288         gk20a_mem_wr32(c->userd_cpu_va, ram_userd_gp_get_w(), 0);
 289         gk20a_mem_wr32(c->userd_cpu_va, ram_userd_gp_put_w(), 0);
 290
 291         return 0;
 292 }
 293
 294 static void channel_gk20a_bind(struct channel_gk20a *ch_gk20a)
 295 {
 296         struct gk20a *g = ch_gk20a->g;
 297         struct fifo_gk20a *f = &g->fifo;
 298         struct fifo_engine_info_gk20a *engine_info =
 299                 f->engine_info + ENGINE_GR_GK20A;
 300
 301         u32 inst_ptr = ch_gk20a->inst_block.cpu_pa
 302                 >> ram_in_base_shift_v();
 303
 304         gk20a_dbg_info("bind channel %d inst ptr 0x%08x",
 305                 ch_gk20a->hw_chid, inst_ptr);
 306
 307         ch_gk20a->bound = true;
 308
 309         gk20a_writel(g, ccsr_channel_r(ch_gk20a->hw_chid),
 310                 (gk20a_readl(g, ccsr_channel_r(ch_gk20a->hw_chid)) &
 311                  ~ccsr_channel_runlist_f(~0)) |
 312                  ccsr_channel_runlist_f(engine_info->runlist_id));
 313
 314         gk20a_writel(g, ccsr_channel_inst_r(ch_gk20a->hw_chid),
 315                 ccsr_channel_inst_ptr_f(inst_ptr) |
 316                 ccsr_channel_inst_target_vid_mem_f() |
 317                 ccsr_channel_inst_bind_true_f());
 318
 319         gk20a_writel(g, ccsr_channel_r(ch_gk20a->hw_chid),
 320                 (gk20a_readl(g, ccsr_channel_r(ch_gk20a->hw_chid)) &
 321                  ~ccsr_channel_enable_set_f(~0)) |
 322                  ccsr_channel_enable_set_true_f());
 323 }
 324
 325 static void channel_gk20a_unbind(struct channel_gk20a *ch_gk20a)
 326 {
 327         struct gk20a *g = ch_gk20a->g;
 328
 329         gk20a_dbg_fn("");
 330
 331         if (ch_gk20a->bound)
 332                 gk20a_writel(g, ccsr_channel_inst_r(ch_gk20a->hw_chid),
 333                         ccsr_channel_inst_ptr_f(0) |
 334                         ccsr_channel_inst_bind_false_f());
 335
 336         ch_gk20a->bound = false;
 337
 338         /*
 339          * if we are agrressive then we can destroy the syncpt
 340          * resource at this point
 341          * if not, then it will be destroyed at channel_free()
 342          */
 343         if (ch_gk20a->sync && ch_gk20a->sync->syncpt_aggressive_destroy) {
 344                 ch_gk20a->sync->destroy(ch_gk20a->sync);
 345                 ch_gk20a->sync = NULL;
 346         }
 347 }
 348
 349 static int channel_gk20a_alloc_inst(struct gk20a *g,
 350                                 struct channel_gk20a *ch)
 351 {
 352         struct device *d = dev_from_gk20a(g);
 353         int err = 0;
 354         dma_addr_t iova;
 355
 356         gk20a_dbg_fn("");
 357
 358         ch->inst_block.size = ram_in_alloc_size_v();
 359         ch->inst_block.cpuva = dma_alloc_coherent(d,
 360                                         ch->inst_block.size,
 361                                         &iova,
 362                                         GFP_KERNEL);
 363         if (!ch->inst_block.cpuva) {
 364                 gk20a_err(d, "%s: memory allocation failed\n", __func__);
 365                 err = -ENOMEM;
 366                 goto clean_up;
 367         }
 368
 369         ch->inst_block.iova = iova;
 370         ch->inst_block.cpu_pa = gk20a_get_phys_from_iova(d,
 371                                                         ch->inst_block.iova);
 372         if (!ch->inst_block.cpu_pa) {
 373                 gk20a_err(d, "%s: failed to get physical address\n", __func__);
 374                 err = -ENOMEM;
 375                 goto clean_up;
 376         }
 377
 378         gk20a_dbg_info("channel %d inst block physical addr: 0x%16llx",
 379                 ch->hw_chid, (u64)ch->inst_block.cpu_pa);
 380
 381         gk20a_dbg_fn("done");
 382         return 0;
 383
 384 clean_up:
 385         gk20a_err(d, "fail");
 386         channel_gk20a_free_inst(g, ch);
 387         return err;
 388 }
 389
 390 static void channel_gk20a_free_inst(struct gk20a *g,
 391                                 struct channel_gk20a *ch)
 392 {
 393         struct device *d = dev_from_gk20a(g);
 394
 395         if (ch->inst_block.cpuva)
 396                 dma_free_coherent(d, ch->inst_block.size,
 397                                 ch->inst_block.cpuva, ch->inst_block.iova);
 398         ch->inst_block.cpuva = NULL;
 399         ch->inst_block.iova = 0;
 400         memset(&ch->inst_block, 0, sizeof(struct inst_desc));
 401 }
 402
 403 static int channel_gk20a_update_runlist(struct channel_gk20a *c, bool add)
 404 {
 405         return gk20a_fifo_update_runlist(c->g, 0, c->hw_chid, add, true);
 406 }
 407
 408 void gk20a_disable_channel_no_update(struct channel_gk20a *ch)
 409 {
 410         /* ensure no fences are pending */
 411         if (ch->sync)
 412                 ch->sync->set_min_eq_max(ch->sync);
 413
 414         /* disable channel */
 415         gk20a_writel(ch->g, ccsr_channel_r(ch->hw_chid),
 416                      gk20a_readl(ch->g,
 417                      ccsr_channel_r(ch->hw_chid)) |
 418                      ccsr_channel_enable_clr_true_f());
 419 }
 420
 421 int gk20a_wait_channel_idle(struct channel_gk20a *ch)
 422 {
 423         bool channel_idle = false;
 424         unsigned long end_jiffies = jiffies +
 425                 msecs_to_jiffies(gk20a_get_gr_idle_timeout(ch->g));
 426
 427         do {
 428                 mutex_lock(&ch->jobs_lock);
 429                 channel_idle = list_empty(&ch->jobs);
 430                 mutex_unlock(&ch->jobs_lock);
 431                 if (channel_idle)
 432                         break;
 433
 434                 usleep_range(1000, 3000);
 435         } while (time_before(jiffies, end_jiffies)
 436                         || !tegra_platform_is_silicon());
 437
 438         if (!channel_idle) {
 439                 gk20a_err(dev_from_gk20a(ch->g), "jobs not freed for channel %d\n",
 440                                 ch->hw_chid);
 441                 return -EBUSY;
 442         }
 443
 444         return 0;
 445 }
 446
 447 void gk20a_disable_channel(struct channel_gk20a *ch,
 448                            bool finish,
 449                            unsigned long finish_timeout)
 450 {
 451         if (finish) {
 452                 int err = gk20a_channel_finish(ch, finish_timeout);
 453                 WARN_ON(err);
 454         }
 455
 456         /* disable the channel from hw and increment syncpoints */
 457         gk20a_disable_channel_no_update(ch);
 458
 459         gk20a_wait_channel_idle(ch);
 460
 461         /* preempt the channel */
 462         gk20a_fifo_preempt_channel(ch->g, ch->hw_chid);
 463
 464         /* remove channel from runlist */
 465         channel_gk20a_update_runlist(ch, false);
 466 }
 467
 468 #if defined(CONFIG_GK20A_CYCLE_STATS)
 469
 470 static void gk20a_free_cycle_stats_buffer(struct channel_gk20a *ch)
 471 {
 472         /* disable existing cyclestats buffer */
 473         mutex_lock(&ch->cyclestate.cyclestate_buffer_mutex);
 474         if (ch->cyclestate.cyclestate_buffer_handler) {
 475                 dma_buf_vunmap(ch->cyclestate.cyclestate_buffer_handler,
 476                                 ch->cyclestate.cyclestate_buffer);
 477                 dma_buf_put(ch->cyclestate.cyclestate_buffer_handler);
 478                 ch->cyclestate.cyclestate_buffer_handler = NULL;
 479                 ch->cyclestate.cyclestate_buffer = NULL;
 480                 ch->cyclestate.cyclestate_buffer_size = 0;
 481         }
 482         mutex_unlock(&ch->cyclestate.cyclestate_buffer_mutex);
 483 }
 484
 485 static int gk20a_channel_cycle_stats(struct channel_gk20a *ch,
 486                        struct nvhost_cycle_stats_args *args)
 487 {
 488         struct dma_buf *dmabuf;
 489         void *virtual_address;
 490
 491         if (args->nvmap_handle && !ch->cyclestate.cyclestate_buffer_handler) {
 492
 493                 /* set up new cyclestats buffer */
 494                 dmabuf = dma_buf_get(args->nvmap_handle);
 495                 if (IS_ERR(dmabuf))
 496                         return PTR_ERR(dmabuf);
 497                 virtual_address = dma_buf_vmap(dmabuf);
 498                 if (!virtual_address)
 499                         return -ENOMEM;
 500
 501                 ch->cyclestate.cyclestate_buffer_handler = dmabuf;
 502                 ch->cyclestate.cyclestate_buffer = virtual_address;
 503                 ch->cyclestate.cyclestate_buffer_size = dmabuf->size;
 504                 return 0;
 505
 506         } else if (!args->nvmap_handle &&
 507                         ch->cyclestate.cyclestate_buffer_handler) {
 508                 gk20a_free_cycle_stats_buffer(ch);
 509                 return 0;
 510
 511         } else if (!args->nvmap_handle &&
 512                         !ch->cyclestate.cyclestate_buffer_handler) {
 513                 /* no requst from GL */
 514                 return 0;
 515
 516         } else {
 517                 pr_err("channel already has cyclestats buffer\n");
 518                 return -EINVAL;
 519         }
 520 }
 521 #endif
 522
 523 static int gk20a_init_error_notifier(struct channel_gk20a *ch,
 524                 struct nvhost_set_error_notifier *args) {
 525         void *va;
 526
 527         struct dma_buf *dmabuf;
 528
 529         if (!args->mem) {
 530                 pr_err("gk20a_init_error_notifier: invalid memory handle\n");
 531                 return -EINVAL;
 532         }
 533
 534         dmabuf = dma_buf_get(args->mem);
 535
 536         if (ch->error_notifier_ref)
 537                 gk20a_free_error_notifiers(ch);
 538
 539         if (IS_ERR(dmabuf)) {
 540                 pr_err("Invalid handle: %d\n", args->mem);
 541                 return -EINVAL;
 542         }
 543         /* map handle */
 544         va = dma_buf_vmap(dmabuf);
 545         if (!va) {
 546                 dma_buf_put(dmabuf);
 547                 pr_err("Cannot map notifier handle\n");
 548                 return -ENOMEM;
 549         }
 550
 551         /* set channel notifiers pointer */
 552         ch->error_notifier_ref = dmabuf;
 553         ch->error_notifier = va + args->offset;
 554         ch->error_notifier_va = va;
 555         memset(ch->error_notifier, 0, sizeof(struct nvhost_notification));
 556         return 0;
 557 }
 558
 559 void gk20a_set_error_notifier(struct channel_gk20a *ch, __u32 error)
 560 {
 561         if (ch->error_notifier_ref) {
 562                 struct timespec time_data;
 563                 u64 nsec;
 564                 getnstimeofday(&time_data);
 565                 nsec = ((u64)time_data.tv_sec) * 1000000000u +
 566                                 (u64)time_data.tv_nsec;
 567                 ch->error_notifier->time_stamp.nanoseconds[0] =
 568                                 (u32)nsec;
 569                 ch->error_notifier->time_stamp.nanoseconds[1] =
 570                                 (u32)(nsec >> 32);
 571                 ch->error_notifier->info32 = error;
 572                 ch->error_notifier->status = 0xffff;
 573                 gk20a_err(dev_from_gk20a(ch->g),
 574                                 "error notifier set to %d\n", error);
 575         }
 576 }
 577
 578 static void gk20a_free_error_notifiers(struct channel_gk20a *ch)
 579 {
 580         if (ch->error_notifier_ref) {
 581                 dma_buf_vunmap(ch->error_notifier_ref, ch->error_notifier_va);
 582                 dma_buf_put(ch->error_notifier_ref);
 583                 ch->error_notifier_ref = 0;
 584                 ch->error_notifier = 0;
 585                 ch->error_notifier_va = 0;
 586         }
 587 }
 588
 589 void gk20a_free_channel(struct channel_gk20a *ch, bool finish)
 590 {
 591         struct gk20a *g = ch->g;
 592         struct device *d = dev_from_gk20a(g);
 593         struct fifo_gk20a *f = &g->fifo;
 594         struct gr_gk20a *gr = &g->gr;
 595         struct vm_gk20a *ch_vm = ch->vm;
 596         unsigned long timeout = gk20a_get_gr_idle_timeout(g);
 597         struct dbg_session_gk20a *dbg_s;
 598
 599         gk20a_dbg_fn("");
 600
 601         /* if engine reset was deferred, perform it now */
 602         mutex_lock(&f->deferred_reset_mutex);
 603         if (g->fifo.deferred_reset_pending) {
 604                 gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, "engine reset was"
 605                            " deferred, running now");
 606                 fifo_gk20a_finish_mmu_fault_handling(g, g->fifo.mmu_fault_engines);
 607                 g->fifo.mmu_fault_engines = 0;
 608                 g->fifo.deferred_reset_pending = false;
 609         }
 610         mutex_unlock(&f->deferred_reset_mutex);
 611
 612         if (!ch->bound)
 613                 return;
 614
 615         if (!gk20a_channel_as_bound(ch))
 616                 goto unbind;
 617
 618         gk20a_dbg_info("freeing bound channel context, timeout=%ld",
 619                         timeout);
 620
 621         gk20a_disable_channel(ch, finish && !ch->has_timedout, timeout);
 622
 623         gk20a_free_error_notifiers(ch);
 624
 625         /* release channel ctx */
 626         gk20a_free_channel_ctx(ch);
 627
 628         gk20a_gr_flush_channel_tlb(gr);
 629
 630         memset(&ch->ramfc, 0, sizeof(struct mem_desc_sub));
 631
 632         /* free gpfifo */
 633         if (ch->gpfifo.gpu_va)
 634                 gk20a_gmmu_unmap(ch_vm, ch->gpfifo.gpu_va,
 635                         ch->gpfifo.size, gk20a_mem_flag_none);
 636         if (ch->gpfifo.cpu_va)
 637                 dma_free_coherent(d, ch->gpfifo.size,
 638                         ch->gpfifo.cpu_va, ch->gpfifo.iova);
 639         ch->gpfifo.cpu_va = NULL;
 640         ch->gpfifo.iova = 0;
 641
 642         memset(&ch->gpfifo, 0, sizeof(struct gpfifo_desc));
 643
 644 #if defined(CONFIG_GK20A_CYCLE_STATS)
 645         gk20a_free_cycle_stats_buffer(ch);
 646 #endif
 647
 648         channel_gk20a_free_priv_cmdbuf(ch);
 649
 650         if (ch->sync) {
 651                 ch->sync->destroy(ch->sync);
 652                 ch->sync = NULL;
 653         }
 654
 655         /* release channel binding to the as_share */
 656         gk20a_as_release_share(ch_vm->as_share);
 657
 658 unbind:
 659         channel_gk20a_unbind(ch);
 660         channel_gk20a_free_inst(g, ch);
 661
 662         ch->vpr = false;
 663         ch->vm = NULL;
 664         WARN_ON(ch->sync);
 665
 666         /* unlink all debug sessions */
 667         mutex_lock(&ch->dbg_s_lock);
 668
 669         list_for_each_entry(dbg_s, &ch->dbg_s_list, dbg_s_list_node) {
 670                 dbg_s->ch = NULL;
 671                 list_del_init(&dbg_s->dbg_s_list_node);
 672         }
 673
 674         mutex_unlock(&ch->dbg_s_lock);
 675
 676         /* ALWAYS last */
 677         release_used_channel(f, ch);
 678 }
 679
 680 int gk20a_channel_release(struct inode *inode, struct file *filp)
 681 {
 682         struct channel_gk20a *ch = (struct channel_gk20a *)filp->private_data;
 683         struct gk20a *g = ch->g;
 684         int err;
 685
 686         trace_gk20a_channel_release(dev_name(&g->dev->dev));
 687
 688         err = gk20a_busy(ch->g->dev);
 689         if (err) {
 690                 gk20a_err(dev_from_gk20a(g), "failed to release channel %d",
 691                         ch->hw_chid);
 692                 return err;
 693         }
 694         gk20a_free_channel(ch, true);
 695         gk20a_idle(ch->g->dev);
 696
 697         gk20a_put_client(g);
 698         filp->private_data = NULL;
 699         return 0;
 700 }
 701
 702 static struct channel_gk20a *gk20a_open_new_channel(struct gk20a *g)
 703 {
 704         struct fifo_gk20a *f = &g->fifo;
 705         struct channel_gk20a *ch;
 706
 707         ch = acquire_unused_channel(f);
 708         if (ch == NULL) {
 709                 /* TBD: we want to make this virtualizable */
 710                 gk20a_err(dev_from_gk20a(g), "out of hw chids");
 711                 return 0;
 712         }
 713
 714         ch->g = g;
 715
 716         if (channel_gk20a_alloc_inst(g, ch)) {
 717                 ch->in_use = false;
 718                 gk20a_err(dev_from_gk20a(g),
 719                            "failed to open gk20a channel, out of inst mem");
 720
 721                 return 0;
 722         }
 723         g->ops.fifo.bind_channel(ch);
 724         ch->pid = current->pid;
 725
 726         /* reset timeout counter and update timestamp */
 727         ch->timeout_accumulated_ms = 0;
 728         ch->timeout_gpfifo_get = 0;
 729         /* set gr host default timeout */
 730         ch->timeout_ms_max = gk20a_get_gr_idle_timeout(g);
 731         ch->timeout_debug_dump = true;
 732         ch->has_timedout = false;
 733         ch->obj_class = 0;
 734
 735         /* The channel is *not* runnable at this point. It still needs to have
 736          * an address space bound and allocate a gpfifo and grctx. */
 737
 738         init_waitqueue_head(&ch->notifier_wq);
 739         init_waitqueue_head(&ch->semaphore_wq);
 740         init_waitqueue_head(&ch->submit_wq);
 741
 742         return ch;
 743 }
 744
 745 static int __gk20a_channel_open(struct gk20a *g, struct file *filp)
 746 {
 747         int err;
 748         struct channel_gk20a *ch;
 749
 750         trace_gk20a_channel_open(dev_name(&g->dev->dev));
 751
 752         err = gk20a_get_client(g);
 753         if (err) {
 754                 gk20a_err(dev_from_gk20a(g),
 755                         "failed to get client ref");
 756                 return err;
 757         }
 758
 759         err = gk20a_busy(g->dev);
 760         if (err) {
 761                 gk20a_put_client(g);
 762                 gk20a_err(dev_from_gk20a(g), "failed to power on, %d", err);
 763                 return err;
 764         }
 765         ch = gk20a_open_new_channel(g);
 766         gk20a_idle(g->dev);
 767         if (!ch) {
 768                 gk20a_put_client(g);
 769                 gk20a_err(dev_from_gk20a(g),
 770                         "failed to get f");
 771                 return -ENOMEM;
 772         }
 773
 774         filp->private_data = ch;
 775         return 0;
 776 }
 777
 778 int gk20a_channel_open(struct inode *inode, struct file *filp)
 779 {
 780         struct gk20a *g = container_of(inode->i_cdev,
 781                         struct gk20a, channel.cdev);
 782         return __gk20a_channel_open(g, filp);
 783 }
 784
 785 /* allocate private cmd buffer.
 786    used for inserting commands before/after user submitted buffers. */
 787 static int channel_gk20a_alloc_priv_cmdbuf(struct channel_gk20a *c)
 788 {
 789         struct device *d = dev_from_gk20a(c->g);
 790         struct vm_gk20a *ch_vm = c->vm;
 791         struct priv_cmd_queue *q = &c->priv_cmd_q;
 792         struct priv_cmd_entry *e;
 793         u32 i = 0, size;
 794         int err = 0;
 795         struct sg_table *sgt;
 796         dma_addr_t iova;
 797
 798         /* Kernel can insert gpfifos before and after user gpfifos.
 799            Before user gpfifos, kernel inserts fence_wait, which takes
 800            syncpoint_a (2 dwords) + syncpoint_b (2 dwords) = 4 dwords.
 801            After user gpfifos, kernel inserts fence_get, which takes
 802            wfi (2 dwords) + syncpoint_a (2 dwords) + syncpoint_b (2 dwords)
 803            = 6 dwords.
 804            Worse case if kernel adds both of them for every user gpfifo,
 805            max size of priv_cmdbuf is :
 806            (gpfifo entry number * (2 / 3) * (4 + 6) * 4 bytes */
 807         size = roundup_pow_of_two(
 808                 c->gpfifo.entry_num * 2 * 10 * sizeof(u32) / 3);
 809
 810         q->mem.base_cpuva = dma_alloc_coherent(d, size,
 811                                         &iova,
 812                                         GFP_KERNEL);
 813         if (!q->mem.base_cpuva) {
 814                 gk20a_err(d, "%s: memory allocation failed\n", __func__);
 815                 err = -ENOMEM;
 816                 goto clean_up;
 817         }
 818
 819         q->mem.base_iova = iova;
 820         q->mem.size = size;
 821
 822         err = gk20a_get_sgtable(d, &sgt,
 823                         q->mem.base_cpuva, q->mem.base_iova, size);
 824         if (err) {
 825                 gk20a_err(d, "%s: failed to create sg table\n", __func__);
 826                 goto clean_up;
 827         }
 828
 829         memset(q->mem.base_cpuva, 0, size);
 830
 831         q->base_gpuva = gk20a_gmmu_map(ch_vm, &sgt,
 832                                         size,
 833                                         0, /* flags */
 834                                         gk20a_mem_flag_none);
 835         if (!q->base_gpuva) {
 836                 gk20a_err(d, "ch %d : failed to map gpu va"
 837                            "for priv cmd buffer", c->hw_chid);
 838                 err = -ENOMEM;
 839                 goto clean_up_sgt;
 840         }
 841
 842         q->size = q->mem.size / sizeof (u32);
 843
 844         INIT_LIST_HEAD(&q->head);
 845         INIT_LIST_HEAD(&q->free);
 846
 847         /* pre-alloc 25% of priv cmdbuf entries and put them on free list */
 848         for (i = 0; i < q->size / 4; i++) {
 849                 e = kzalloc(sizeof(struct priv_cmd_entry), GFP_KERNEL);
 850                 if (!e) {
 851                         gk20a_err(d, "ch %d: fail to pre-alloc cmd entry",
 852                                 c->hw_chid);
 853                         err = -ENOMEM;
 854                         goto clean_up_sgt;
 855                 }
 856                 e->pre_alloc = true;
 857                 list_add(&e->list, &q->free);
 858         }
 859
 860         gk20a_free_sgtable(&sgt);
 861
 862         return 0;
 863
 864 clean_up_sgt:
 865         gk20a_free_sgtable(&sgt);
 866 clean_up:
 867         channel_gk20a_free_priv_cmdbuf(c);
 868         return err;
 869 }
 870
 871 static void channel_gk20a_free_priv_cmdbuf(struct channel_gk20a *c)
 872 {
 873         struct device *d = dev_from_gk20a(c->g);
 874         struct vm_gk20a *ch_vm = c->vm;
 875         struct priv_cmd_queue *q = &c->priv_cmd_q;
 876         struct priv_cmd_entry *e;
 877         struct list_head *pos, *tmp, *head;
 878
 879         if (q->size == 0)
 880                 return;
 881
 882         if (q->base_gpuva)
 883                 gk20a_gmmu_unmap(ch_vm, q->base_gpuva,
 884                                 q->mem.size, gk20a_mem_flag_none);
 885         if (q->mem.base_cpuva)
 886                 dma_free_coherent(d, q->mem.size,
 887                         q->mem.base_cpuva, q->mem.base_iova);
 888         q->mem.base_cpuva = NULL;
 889         q->mem.base_iova = 0;
 890
 891         /* free used list */
 892         head = &q->head;
 893         list_for_each_safe(pos, tmp, head) {
 894                 e = container_of(pos, struct priv_cmd_entry, list);
 895                 free_priv_cmdbuf(c, e);
 896         }
 897
 898         /* free free list */
 899         head = &q->free;
 900         list_for_each_safe(pos, tmp, head) {
 901                 e = container_of(pos, struct priv_cmd_entry, list);
 902                 e->pre_alloc = false;
 903                 free_priv_cmdbuf(c, e);
 904         }
 905
 906         memset(q, 0, sizeof(struct priv_cmd_queue));
 907 }
 908
 909 /* allocate a cmd buffer with given size. size is number of u32 entries */
 910 int gk20a_channel_alloc_priv_cmdbuf(struct channel_gk20a *c, u32 orig_size,
 911                              struct priv_cmd_entry **entry)
 912 {
 913         struct priv_cmd_queue *q = &c->priv_cmd_q;
 914         struct priv_cmd_entry *e;
 915         struct list_head *node;
 916         u32 free_count;
 917         u32 size = orig_size;
 918         bool no_retry = false;
 919
 920         gk20a_dbg_fn("size %d", orig_size);
 921
 922         *entry = NULL;
 923
 924         /* if free space in the end is less than requested, increase the size
 925          * to make the real allocated space start from beginning. */
 926         if (q->put + size > q->size)
 927                 size = orig_size + (q->size - q->put);
 928
 929         gk20a_dbg_info("ch %d: priv cmd queue get:put %d:%d",
 930                         c->hw_chid, q->get, q->put);
 931
 932 TRY_AGAIN:
 933         free_count = (q->size - (q->put - q->get) - 1) % q->size;
 934
 935         if (size > free_count) {
 936                 if (!no_retry) {
 937                         recycle_priv_cmdbuf(c);
 938                         no_retry = true;
 939                         goto TRY_AGAIN;
 940                 } else
 941                         return -EAGAIN;
 942         }
 943
 944         if (unlikely(list_empty(&q->free))) {
 945
 946                 gk20a_dbg_info("ch %d: run out of pre-alloc entries",
 947                         c->hw_chid);
 948
 949                 e = kzalloc(sizeof(struct priv_cmd_entry), GFP_KERNEL);
 950                 if (!e) {
 951                         gk20a_err(dev_from_gk20a(c->g),
 952                                 "ch %d: fail to allocate priv cmd entry",
 953                                 c->hw_chid);
 954                         return -ENOMEM;
 955                 }
 956         } else  {
 957                 node = q->free.next;
 958                 list_del(node);
 959                 e = container_of(node, struct priv_cmd_entry, list);
 960         }
 961
 962         e->size = orig_size;
 963         e->gp_get = c->gpfifo.get;
 964         e->gp_put = c->gpfifo.put;
 965         e->gp_wrap = c->gpfifo.wrap;
 966
 967         /* if we have increased size to skip free space in the end, set put
 968            to beginning of cmd buffer (0) + size */
 969         if (size != orig_size) {
 970                 e->ptr = q->mem.base_cpuva;
 971                 e->gva = q->base_gpuva;
 972                 q->put = orig_size;
 973         } else {
 974                 e->ptr = q->mem.base_cpuva + q->put;
 975                 e->gva = q->base_gpuva + q->put * sizeof(u32);
 976                 q->put = (q->put + orig_size) & (q->size - 1);
 977         }
 978
 979         /* we already handled q->put + size > q->size so BUG_ON this */
 980         BUG_ON(q->put > q->size);
 981
 982         /* add new entry to head since we free from head */
 983         list_add(&e->list, &q->head);
 984
 985         *entry = e;
 986
 987         gk20a_dbg_fn("done");
 988
 989         return 0;
 990 }
 991
 992 /* Don't call this to free an explict cmd entry.
 993  * It doesn't update priv_cmd_queue get/put */
 994 static void free_priv_cmdbuf(struct channel_gk20a *c,
 995                              struct priv_cmd_entry *e)
 996 {
 997         struct priv_cmd_queue *q = &c->priv_cmd_q;
 998
 999         if (!e)
1000                 return;
1001
1002         list_del(&e->list);
1003
1004         if (unlikely(!e->pre_alloc))
1005                 kfree(e);
1006         else {
1007                 memset(e, 0, sizeof(struct priv_cmd_entry));
1008                 e->pre_alloc = true;
1009                 list_add(&e->list, &q->free);
1010         }
1011 }
1012
1013 /* free entries if they're no longer being used */
1014 static void recycle_priv_cmdbuf(struct channel_gk20a *c)
1015 {
1016         struct priv_cmd_queue *q = &c->priv_cmd_q;
1017         struct priv_cmd_entry *e, *tmp;
1018         struct list_head *head = &q->head;
1019         bool wrap_around, found = false;
1020
1021         gk20a_dbg_fn("");
1022
1023         /* Find the most recent free entry. Free it and everything before it */
1024         list_for_each_entry(e, head, list) {
1025
1026                 gk20a_dbg_info("ch %d: cmd entry get:put:wrap %d:%d:%d "
1027                         "curr get:put:wrap %d:%d:%d",
1028                         c->hw_chid, e->gp_get, e->gp_put, e->gp_wrap,
1029                         c->gpfifo.get, c->gpfifo.put, c->gpfifo.wrap);
1030
1031                 wrap_around = (c->gpfifo.wrap != e->gp_wrap);
1032                 if (e->gp_get < e->gp_put) {
1033                         if (c->gpfifo.get >= e->gp_put ||
1034                             wrap_around) {
1035                                 found = true;
1036                                 break;
1037                         } else
1038                                 e->gp_get = c->gpfifo.get;
1039                 } else if (e->gp_get > e->gp_put) {
1040                         if (wrap_around &&
1041                             c->gpfifo.get >= e->gp_put) {
1042                                 found = true;
1043                                 break;
1044                         } else
1045                                 e->gp_get = c->gpfifo.get;
1046                 }
1047         }
1048
1049         if (found)
1050                 q->get = (e->ptr - q->mem.base_cpuva) + e->size;
1051         else {
1052                 gk20a_dbg_info("no free entry recycled");
1053                 return;
1054         }
1055
1056         list_for_each_entry_safe_continue(e, tmp, head, list) {
1057                 free_priv_cmdbuf(c, e);
1058         }
1059
1060         gk20a_dbg_fn("done");
1061 }
1062
1063
1064 static int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c,
1065                                       struct nvhost_alloc_gpfifo_args *args)
1066 {
1067         struct gk20a *g = c->g;
1068         struct device *d = dev_from_gk20a(g);
1069         struct vm_gk20a *ch_vm;
1070         u32 gpfifo_size;
1071         int err = 0;
1072         struct sg_table *sgt;
1073         dma_addr_t iova;
1074
1075         /* Kernel can insert one extra gpfifo entry before user submitted gpfifos
1076            and another one after, for internal usage. Triple the requested size. */
1077         gpfifo_size = roundup_pow_of_two(args->num_entries * 3);
1078
1079         if (args->flags & NVHOST_ALLOC_GPFIFO_FLAGS_VPR_ENABLED)
1080                 c->vpr = true;
1081
1082         /* an address space needs to have been bound at this point.   */
1083         if (!gk20a_channel_as_bound(c)) {
1084                 gk20a_err(d,
1085                             "not bound to an address space at time of gpfifo"
1086                             " allocation.  Attempting to create and bind to"
1087                             " one...");
1088                 return -EINVAL;
1089         }
1090         ch_vm = c->vm;
1091
1092         c->cmds_pending = false;
1093         c->last_submit_fence.valid = false;
1094
1095         c->ramfc.offset = 0;
1096         c->ramfc.size = ram_in_ramfc_s() / 8;
1097
1098         if (c->gpfifo.cpu_va) {
1099                 gk20a_err(d, "channel %d :"
1100                            "gpfifo already allocated", c->hw_chid);
1101                 return -EEXIST;
1102         }
1103
1104         c->gpfifo.size = gpfifo_size * sizeof(struct gpfifo);
1105         c->gpfifo.cpu_va = (struct gpfifo *)dma_alloc_coherent(d,
1106                                                 c->gpfifo.size,
1107                                                 &iova,
1108                                                 GFP_KERNEL);
1109         if (!c->gpfifo.cpu_va) {
1110                 gk20a_err(d, "%s: memory allocation failed\n", __func__);
1111                 err = -ENOMEM;
1112                 goto clean_up;
1113         }
1114
1115         c->gpfifo.iova = iova;
1116         c->gpfifo.entry_num = gpfifo_size;
1117
1118         c->gpfifo.get = c->gpfifo.put = 0;
1119
1120         err = gk20a_get_sgtable(d, &sgt,
1121                         c->gpfifo.cpu_va, c->gpfifo.iova, c->gpfifo.size);
1122         if (err) {
1123                 gk20a_err(d, "%s: failed to allocate sg table\n", __func__);
1124                 goto clean_up;
1125         }
1126
1127         c->gpfifo.gpu_va = gk20a_gmmu_map(ch_vm,
1128                                         &sgt,
1129                                         c->gpfifo.size,
1130                                         0, /* flags */
1131                                         gk20a_mem_flag_none);
1132         if (!c->gpfifo.gpu_va) {
1133                 gk20a_err(d, "channel %d : failed to map"
1134                            " gpu_va for gpfifo", c->hw_chid);
1135                 err = -ENOMEM;
1136                 goto clean_up_sgt;
1137         }
1138
1139         gk20a_dbg_info("channel %d : gpfifo_base 0x%016llx, size %d",
1140                 c->hw_chid, c->gpfifo.gpu_va, c->gpfifo.entry_num);
1141
1142         channel_gk20a_setup_ramfc(c, c->gpfifo.gpu_va, c->gpfifo.entry_num);
1143
1144         channel_gk20a_setup_userd(c);
1145         channel_gk20a_commit_userd(c);
1146
1147         /* TBD: setup engine contexts */
1148
1149         err = channel_gk20a_alloc_priv_cmdbuf(c);
1150         if (err)
1151                 goto clean_up_unmap;
1152
1153         err = channel_gk20a_update_runlist(c, true);
1154         if (err)
1155                 goto clean_up_unmap;
1156
1157         gk20a_free_sgtable(&sgt);
1158
1159         gk20a_dbg_fn("done");
1160         return 0;
1161
1162 clean_up_unmap:
1163         gk20a_gmmu_unmap(ch_vm, c->gpfifo.gpu_va,
1164                 c->gpfifo.size, gk20a_mem_flag_none);
1165 clean_up_sgt:
1166         gk20a_free_sgtable(&sgt);
1167 clean_up:
1168         dma_free_coherent(d, c->gpfifo.size,
1169                 c->gpfifo.cpu_va, c->gpfifo.iova);
1170         c->gpfifo.cpu_va = NULL;
1171         c->gpfifo.iova = 0;
1172         memset(&c->gpfifo, 0, sizeof(struct gpfifo_desc));
1173         gk20a_err(d, "fail");
1174         return err;
1175 }
1176
1177 static inline int wfi_cmd_size(void)
1178 {
1179         return 2;
1180 }
1181 void add_wfi_cmd(struct priv_cmd_entry *cmd, int *i)
1182 {
1183         /* wfi */
1184         cmd->ptr[(*i)++] = 0x2001001E;
1185         /* handle, ignored */
1186         cmd->ptr[(*i)++] = 0x00000000;
1187 }
1188
1189 static inline bool check_gp_put(struct gk20a *g,
1190                                 struct channel_gk20a *c)
1191 {
1192         u32 put;
1193         /* gp_put changed unexpectedly since last update? */
1194         put = gk20a_bar1_readl(g,
1195                c->userd_gpu_va + 4 * ram_userd_gp_put_w());
1196         if (c->gpfifo.put != put) {
1197                 /*TBD: BUG_ON/teardown on this*/
1198                 gk20a_err(dev_from_gk20a(g), "gp_put changed unexpectedly "
1199                            "since last update");
1200                 c->gpfifo.put = put;
1201                 return false; /* surprise! */
1202         }
1203         return true; /* checked out ok */
1204 }
1205
1206 /* Update with this periodically to determine how the gpfifo is draining. */
1207 static inline u32 update_gp_get(struct gk20a *g,
1208                                 struct channel_gk20a *c)
1209 {
1210         u32 new_get = gk20a_bar1_readl(g,
1211                 c->userd_gpu_va + sizeof(u32) * ram_userd_gp_get_w());
1212         if (new_get < c->gpfifo.get)
1213                 c->gpfifo.wrap = !c->gpfifo.wrap;
1214         c->gpfifo.get = new_get;
1215         return new_get;
1216 }
1217
1218 static inline u32 gp_free_count(struct channel_gk20a *c)
1219 {
1220         return (c->gpfifo.entry_num - (c->gpfifo.put - c->gpfifo.get) - 1) %
1221                 c->gpfifo.entry_num;
1222 }
1223
1224 bool gk20a_channel_update_and_check_timeout(struct channel_gk20a *ch,
1225                 u32 timeout_delta_ms)
1226 {
1227         u32 gpfifo_get = update_gp_get(ch->g, ch);
1228         /* Count consequent timeout isr */
1229         if (gpfifo_get == ch->timeout_gpfifo_get) {
1230                 /* we didn't advance since previous channel timeout check */
1231                 ch->timeout_accumulated_ms += timeout_delta_ms;
1232         } else {
1233                 /* first timeout isr encountered */
1234                 ch->timeout_accumulated_ms = timeout_delta_ms;
1235         }
1236
1237         ch->timeout_gpfifo_get = gpfifo_get;
1238
1239         return ch->g->timeouts_enabled &&
1240                 ch->timeout_accumulated_ms > ch->timeout_ms_max;
1241 }
1242
1243
1244 /* Issue a syncpoint increment *preceded* by a wait-for-idle
1245  * command.  All commands on the channel will have been
1246  * consumed at the time the fence syncpoint increment occurs.
1247  */
1248 static int gk20a_channel_submit_wfi(struct channel_gk20a *c)
1249 {
1250         struct priv_cmd_entry *cmd = NULL;
1251         struct gk20a *g = c->g;
1252         u32 free_count;
1253         int err;
1254
1255         if (c->has_timedout)
1256                 return -ETIMEDOUT;
1257
1258         if (!c->sync) {
1259                 c->sync = gk20a_channel_sync_create(c);
1260                 if (!c->sync)
1261                         return -ENOMEM;
1262         }
1263
1264         update_gp_get(g, c);
1265         free_count = gp_free_count(c);
1266         if (unlikely(!free_count)) {
1267                 gk20a_err(dev_from_gk20a(g),
1268                            "not enough gpfifo space");
1269                 return -EAGAIN;
1270         }
1271
1272         err = c->sync->incr_wfi(c->sync, &cmd, &c->last_submit_fence);
1273         if (unlikely(err))
1274                 return err;
1275
1276         WARN_ON(!c->last_submit_fence.wfi);
1277
1278         c->gpfifo.cpu_va[c->gpfifo.put].entry0 = u64_lo32(cmd->gva);
1279         c->gpfifo.cpu_va[c->gpfifo.put].entry1 = u64_hi32(cmd->gva) |
1280                 pbdma_gp_entry1_length_f(cmd->size);
1281
1282         c->gpfifo.put = (c->gpfifo.put + 1) & (c->gpfifo.entry_num - 1);
1283
1284         /* save gp_put */
1285         cmd->gp_put = c->gpfifo.put;
1286
1287         gk20a_bar1_writel(g,
1288                 c->userd_gpu_va + 4 * ram_userd_gp_put_w(),
1289                 c->gpfifo.put);
1290
1291         gk20a_dbg_info("post-submit put %d, get %d, size %d",
1292                 c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
1293
1294         return 0;
1295 }
1296
1297 static u32 get_gp_free_count(struct channel_gk20a *c)
1298 {
1299         update_gp_get(c->g, c);
1300         return gp_free_count(c);
1301 }
1302
1303 static void trace_write_pushbuffer(struct channel_gk20a *c, struct gpfifo *g)
1304 {
1305         void *mem = NULL;
1306         unsigned int words;
1307         u64 offset;
1308         struct dma_buf *dmabuf = NULL;
1309
1310         if (gk20a_debug_trace_cmdbuf) {
1311                 u64 gpu_va = (u64)g->entry0 |
1312                         (u64)((u64)pbdma_gp_entry1_get_hi_v(g->entry1) << 32);
1313                 int err;
1314
1315                 words = pbdma_gp_entry1_length_v(g->entry1);
1316                 err = gk20a_vm_find_buffer(c->vm, gpu_va, &dmabuf, &offset);
1317                 if (!err)
1318                         mem = dma_buf_vmap(dmabuf);
1319         }
1320
1321         if (mem) {
1322                 u32 i;
1323                 /*
1324                  * Write in batches of 128 as there seems to be a limit
1325                  * of how much you can output to ftrace at once.
1326                  */
1327                 for (i = 0; i < words; i += 128U) {
1328                         trace_gk20a_push_cmdbuf(
1329                                 c->g->dev->name,
1330                                 0,
1331                                 min(words - i, 128U),
1332                                 offset + i * sizeof(u32),
1333                                 mem);
1334                 }
1335                 dma_buf_vunmap(dmabuf, mem);
1336         }
1337 }
1338
1339 static int gk20a_channel_add_job(struct channel_gk20a *c,
1340                                  struct gk20a_channel_fence *fence)
1341 {
1342         struct vm_gk20a *vm = c->vm;
1343         struct channel_gk20a_job *job = NULL;
1344         struct mapped_buffer_node **mapped_buffers = NULL;
1345         int err = 0, num_mapped_buffers;
1346
1347         /* job needs reference to this vm */
1348         gk20a_vm_get(vm);
1349
1350         err = gk20a_vm_get_buffers(vm, &mapped_buffers, &num_mapped_buffers);
1351         if (err) {
1352                 gk20a_vm_put(vm);
1353                 return err;
1354         }
1355
1356         job = kzalloc(sizeof(*job), GFP_KERNEL);
1357         if (!job) {
1358                 gk20a_vm_put_buffers(vm, mapped_buffers, num_mapped_buffers);
1359                 gk20a_vm_put(vm);
1360                 return -ENOMEM;
1361         }
1362
1363         job->num_mapped_buffers = num_mapped_buffers;
1364         job->mapped_buffers = mapped_buffers;
1365         job->fence = *fence;
1366
1367         mutex_lock(&c->jobs_lock);
1368         list_add_tail(&job->list, &c->jobs);
1369         mutex_unlock(&c->jobs_lock);
1370
1371         return 0;
1372 }
1373
1374 void gk20a_channel_update(struct channel_gk20a *c, int nr_completed)
1375 {
1376         struct vm_gk20a *vm = c->vm;
1377         struct channel_gk20a_job *job, *n;
1378
1379         wake_up(&c->submit_wq);
1380
1381         mutex_lock(&c->jobs_lock);
1382         list_for_each_entry_safe(job, n, &c->jobs, list) {
1383                 bool completed = WARN_ON(!c->sync) ||
1384                         c->sync->is_expired(c->sync, &job->fence);
1385                 if (!completed)
1386                         break;
1387
1388                 gk20a_vm_put_buffers(vm, job->mapped_buffers,
1389                                 job->num_mapped_buffers);
1390
1391                 /* job is done. release its reference to vm */
1392                 gk20a_vm_put(vm);
1393
1394                 list_del_init(&job->list);
1395                 kfree(job);
1396                 gk20a_idle(c->g->dev);
1397         }
1398         mutex_unlock(&c->jobs_lock);
1399 }
1400
1401 static int gk20a_submit_channel_gpfifo(struct channel_gk20a *c,
1402                                 struct nvhost_gpfifo *gpfifo,
1403                                 u32 num_entries,
1404                                 struct nvhost_fence *fence,
1405                                 u32 flags)
1406 {
1407         struct gk20a *g = c->g;
1408         struct device *d = dev_from_gk20a(g);
1409         int err = 0;
1410         int i;
1411         struct priv_cmd_entry *wait_cmd = NULL;
1412         struct priv_cmd_entry *incr_cmd = NULL;
1413         /* we might need two extra gpfifo entries - one for pre fence
1414          * and one for post fence. */
1415         const int extra_entries = 2;
1416         bool need_wfi = !(flags & NVHOST_SUBMIT_GPFIFO_FLAGS_SUPPRESS_WFI);
1417
1418         if (c->has_timedout)
1419                 return -ETIMEDOUT;
1420
1421         if ((flags & (NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT |
1422                       NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_GET)) &&
1423             !fence)
1424                 return -EINVAL;
1425
1426         if (!c->sync) {
1427                 c->sync = gk20a_channel_sync_create(c);
1428                 if (!c->sync)
1429                         return -ENOMEM;
1430         }
1431
1432 #ifdef CONFIG_DEBUG_FS
1433         /* update debug settings */
1434         if (g->ops.ltc.sync_debugfs)
1435                 g->ops.ltc.sync_debugfs(g);
1436 #endif
1437
1438         gk20a_dbg_info("channel %d", c->hw_chid);
1439
1440         /* gk20a_channel_update releases this ref. */
1441         err = gk20a_busy(g->dev);
1442         if (err) {
1443                 gk20a_err(d, "failed to host gk20a to submit gpfifo");
1444                 return err;
1445         }
1446
1447         trace_gk20a_channel_submit_gpfifo(c->g->dev->name,
1448                                           c->hw_chid,
1449                                           num_entries,
1450                                           flags,
1451                                           fence->syncpt_id, fence->value);
1452         check_gp_put(g, c);
1453         update_gp_get(g, c);
1454
1455         gk20a_dbg_info("pre-submit put %d, get %d, size %d",
1456                 c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
1457
1458         /* Invalidate tlb if it's dirty...                                   */
1459         /* TBD: this should be done in the cmd stream, not with PRIs.        */
1460         /* We don't know what context is currently running...                */
1461         /* Note also: there can be more than one context associated with the */
1462         /* address space (vm).   */
1463         gk20a_mm_tlb_invalidate(c->vm);
1464
1465         /* Make sure we have enough space for gpfifo entries. If not,
1466          * wait for signals from completed submits */
1467         if (gp_free_count(c) < num_entries + extra_entries) {
1468                 err = wait_event_interruptible(c->submit_wq,
1469                         get_gp_free_count(c) >= num_entries + extra_entries ||
1470                         c->has_timedout);
1471         }
1472
1473         if (c->has_timedout) {
1474                 err = -ETIMEDOUT;
1475                 goto clean_up;
1476         }
1477
1478         if (err) {
1479                 gk20a_err(d, "not enough gpfifo space");
1480                 err = -EAGAIN;
1481                 goto clean_up;
1482         }
1483
1484         /*
1485          * optionally insert syncpt wait in the beginning of gpfifo submission
1486          * when user requested and the wait hasn't expired.
1487          * validate that the id makes sense, elide if not
1488          * the only reason this isn't being unceremoniously killed is to
1489          * keep running some tests which trigger this condition
1490          */
1491         if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_WAIT) {
1492                 if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE)
1493                         err = c->sync->wait_fd(c->sync, fence->syncpt_id,
1494                                         &wait_cmd);
1495                 else
1496                         err = c->sync->wait_syncpt(c->sync, fence->syncpt_id,
1497                                         fence->value, &wait_cmd);
1498         }
1499         if (err)
1500                 goto clean_up;
1501
1502
1503         /* always insert syncpt increment at end of gpfifo submission
1504            to keep track of method completion for idle railgating */
1505         if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_GET &&
1506                         flags & NVHOST_SUBMIT_GPFIFO_FLAGS_SYNC_FENCE)
1507                 err = c->sync->incr_user_fd(c->sync, &incr_cmd,
1508                                             &c->last_submit_fence,
1509                                             need_wfi,
1510                                             &fence->syncpt_id);
1511         else if (flags & NVHOST_SUBMIT_GPFIFO_FLAGS_FENCE_GET)
1512                 err = c->sync->incr_user_syncpt(c->sync, &incr_cmd,
1513                                                 &c->last_submit_fence,
1514                                                 need_wfi,
1515                                                 &fence->syncpt_id,
1516                                                 &fence->value);
1517         else
1518                 err = c->sync->incr(c->sync, &incr_cmd,
1519                                     &c->last_submit_fence);
1520         if (err)
1521                 goto clean_up;
1522
1523         if (wait_cmd) {
1524                 c->gpfifo.cpu_va[c->gpfifo.put].entry0 =
1525                         u64_lo32(wait_cmd->gva);
1526                 c->gpfifo.cpu_va[c->gpfifo.put].entry1 =
1527                         u64_hi32(wait_cmd->gva) |
1528                         pbdma_gp_entry1_length_f(wait_cmd->size);
1529                 trace_gk20a_push_cmdbuf(c->g->dev->name,
1530                         0, wait_cmd->size, 0, wait_cmd->ptr);
1531
1532                 c->gpfifo.put = (c->gpfifo.put + 1) &
1533                         (c->gpfifo.entry_num - 1);
1534
1535                 /* save gp_put */
1536                 wait_cmd->gp_put = c->gpfifo.put;
1537         }
1538
1539         for (i = 0; i < num_entries; i++) {
1540                 c->gpfifo.cpu_va[c->gpfifo.put].entry0 =
1541                         gpfifo[i].entry0; /* cmd buf va low 32 */
1542                 c->gpfifo.cpu_va[c->gpfifo.put].entry1 =
1543                         gpfifo[i].entry1; /* cmd buf va high 32 | words << 10 */
1544                 trace_write_pushbuffer(c, &c->gpfifo.cpu_va[c->gpfifo.put]);
1545                 c->gpfifo.put = (c->gpfifo.put + 1) &
1546                         (c->gpfifo.entry_num - 1);
1547         }
1548
1549         if (incr_cmd) {
1550                 c->gpfifo.cpu_va[c->gpfifo.put].entry0 =
1551                         u64_lo32(incr_cmd->gva);
1552                 c->gpfifo.cpu_va[c->gpfifo.put].entry1 =
1553                         u64_hi32(incr_cmd->gva) |
1554                         pbdma_gp_entry1_length_f(incr_cmd->size);
1555                 trace_gk20a_push_cmdbuf(c->g->dev->name,
1556                         0, incr_cmd->size, 0, incr_cmd->ptr);
1557
1558                 c->gpfifo.put = (c->gpfifo.put + 1) &
1559                         (c->gpfifo.entry_num - 1);
1560
1561                 /* save gp_put */
1562                 incr_cmd->gp_put = c->gpfifo.put;
1563         }
1564
1565         trace_gk20a_channel_submitted_gpfifo(c->g->dev->name,
1566                                              c->hw_chid,
1567                                              num_entries,
1568                                              flags,
1569                                              fence->syncpt_id, fence->value);
1570
1571         /* TODO! Check for errors... */
1572         gk20a_channel_add_job(c, &c->last_submit_fence);
1573
1574         c->cmds_pending = true;
1575         gk20a_bar1_writel(g,
1576                 c->userd_gpu_va + 4 * ram_userd_gp_put_w(),
1577                 c->gpfifo.put);
1578
1579         gk20a_dbg_info("post-submit put %d, get %d, size %d",
1580                 c->gpfifo.put, c->gpfifo.get, c->gpfifo.entry_num);
1581
1582         gk20a_dbg_fn("done");
1583         return err;
1584
1585 clean_up:
1586         gk20a_err(d, "fail");
1587         free_priv_cmdbuf(c, wait_cmd);
1588         free_priv_cmdbuf(c, incr_cmd);
1589         gk20a_idle(g->dev);
1590         return err;
1591 }
1592
1593 void gk20a_remove_channel_support(struct channel_gk20a *c)
1594 {
1595
1596 }
1597
1598 int gk20a_init_channel_support(struct gk20a *g, u32 chid)
1599 {
1600         struct channel_gk20a *c = g->fifo.channel+chid;
1601         c->g = g;
1602         c->in_use = false;
1603         c->hw_chid = chid;
1604         c->bound = false;
1605         c->remove_support = gk20a_remove_channel_support;
1606         mutex_init(&c->jobs_lock);
1607         INIT_LIST_HEAD(&c->jobs);
1608 #if defined(CONFIG_GK20A_CYCLE_STATS)
1609         mutex_init(&c->cyclestate.cyclestate_buffer_mutex);
1610 #endif
1611         INIT_LIST_HEAD(&c->dbg_s_list);
1612         mutex_init(&c->dbg_s_lock);
1613
1614         return 0;
1615 }
1616
1617 int gk20a_channel_finish(struct channel_gk20a *ch, unsigned long timeout)
1618 {
1619         int err = 0;
1620
1621         if (!ch->cmds_pending)
1622                 return 0;
1623
1624         /* Do not wait for a timedout channel */
1625         if (ch->has_timedout)
1626                 return -ETIMEDOUT;
1627
1628         if (!(ch->last_submit_fence.valid && ch->last_submit_fence.wfi)) {
1629                 gk20a_dbg_fn("issuing wfi, incr to finish the channel");
1630                 err = gk20a_channel_submit_wfi(ch);
1631         }
1632         if (err)
1633                 return err;
1634
1635         BUG_ON(!(ch->last_submit_fence.valid && ch->last_submit_fence.wfi));
1636
1637         gk20a_dbg_fn("waiting for channel to finish thresh:%d",
1638                       ch->last_submit_fence.thresh);
1639
1640         err = ch->sync->wait_cpu(ch->sync, &ch->last_submit_fence, timeout);
1641         if (WARN_ON(err))
1642                 dev_warn(dev_from_gk20a(ch->g),
1643                          "timed out waiting for gk20a channel to finish");
1644         else
1645                 ch->cmds_pending = false;
1646
1647         return err;
1648 }
1649
1650 static int gk20a_channel_wait_semaphore(struct channel_gk20a *ch,
1651                                         ulong id, u32 offset,
1652                                         u32 payload, long timeout)
1653 {
1654         struct platform_device *pdev = ch->g->dev;
1655         struct dma_buf *dmabuf;
1656         void *data;
1657         u32 *semaphore;
1658         int ret = 0;
1659         long remain;
1660
1661         /* do not wait if channel has timed out */
1662         if (ch->has_timedout)
1663                 return -ETIMEDOUT;
1664
1665         dmabuf = dma_buf_get(id);
1666         if (IS_ERR(dmabuf)) {
1667                 gk20a_err(&pdev->dev, "invalid notifier nvmap handle 0x%lx",
1668                            id);
1669                 return -EINVAL;
1670         }
1671
1672         data = dma_buf_kmap(dmabuf, offset >> PAGE_SHIFT);
1673         if (!data) {
1674                 gk20a_err(&pdev->dev, "failed to map notifier memory");
1675                 ret = -EINVAL;
1676                 goto cleanup_put;
1677         }
1678
1679         semaphore = data + (offset & ~PAGE_MASK);
1680
1681         remain = wait_event_interruptible_timeout(
1682                         ch->semaphore_wq,
1683                         *semaphore == payload || ch->has_timedout,
1684                         timeout);
1685
1686         if (remain == 0 && *semaphore != payload)
1687                 ret = -ETIMEDOUT;
1688         else if (remain < 0)
1689                 ret = remain;
1690
1691         dma_buf_kunmap(dmabuf, offset >> PAGE_SHIFT, data);
1692 cleanup_put:
1693         dma_buf_put(dmabuf);
1694         return ret;
1695 }
1696
1697 static int gk20a_channel_wait(struct channel_gk20a *ch,
1698                               struct nvhost_wait_args *args)
1699 {
1700         struct device *d = dev_from_gk20a(ch->g);
1701         struct dma_buf *dmabuf;
1702         struct notification *notif;
1703         struct timespec tv;
1704         u64 jiffies;
1705         ulong id;
1706         u32 offset;
1707         unsigned long timeout;
1708         int remain, ret = 0;
1709
1710         gk20a_dbg_fn("");
1711
1712         if (ch->has_timedout)
1713                 return -ETIMEDOUT;
1714
1715         if (args->timeout == NVHOST_NO_TIMEOUT)
1716                 timeout = MAX_SCHEDULE_TIMEOUT;
1717         else
1718                 timeout = (u32)msecs_to_jiffies(args->timeout);
1719
1720         switch (args->type) {
1721         case NVHOST_WAIT_TYPE_NOTIFIER:
1722                 id = args->condition.notifier.nvmap_handle;
1723                 offset = args->condition.notifier.offset;
1724
1725                 dmabuf = dma_buf_get(id);
1726                 if (IS_ERR(dmabuf)) {
1727                         gk20a_err(d, "invalid notifier nvmap handle 0x%lx",
1728                                    id);
1729                         return -EINVAL;
1730                 }
1731
1732                 notif = dma_buf_vmap(dmabuf);
1733                 if (!notif) {
1734                         gk20a_err(d, "failed to map notifier memory");
1735                         return -ENOMEM;
1736                 }
1737
1738                 notif = (struct notification *)((uintptr_t)notif + offset);
1739
1740                 /* user should set status pending before
1741                  * calling this ioctl */
1742                 remain = wait_event_interruptible_timeout(
1743                                 ch->notifier_wq,
1744                                 notif->status == 0 || ch->has_timedout,
1745                                 timeout);
1746
1747                 if (remain == 0 && notif->status != 0) {
1748                         ret = -ETIMEDOUT;
1749                         goto notif_clean_up;
1750                 } else if (remain < 0) {
1751                         ret = -EINTR;
1752                         goto notif_clean_up;
1753                 }
1754
1755                 /* TBD: fill in correct information */
1756                 jiffies = get_jiffies_64();
1757                 jiffies_to_timespec(jiffies, &tv);
1758                 notif->timestamp.nanoseconds[0] = tv.tv_nsec;
1759                 notif->timestamp.nanoseconds[1] = tv.tv_sec;
1760                 notif->info32 = 0xDEADBEEF; /* should be object name */
1761                 notif->info16 = ch->hw_chid; /* should be method offset */
1762
1763 notif_clean_up:
1764                 dma_buf_vunmap(dmabuf, notif);
1765                 return ret;
1766
1767         case NVHOST_WAIT_TYPE_SEMAPHORE:
1768                 ret = gk20a_channel_wait_semaphore(ch,
1769                                 args->condition.semaphore.nvmap_handle,
1770                                 args->condition.semaphore.offset,
1771                                 args->condition.semaphore.payload,
1772                                 timeout);
1773
1774                 break;
1775
1776         default:
1777                 ret = -EINVAL;
1778                 break;
1779         }
1780
1781         return ret;
1782 }
1783
1784 static int gk20a_channel_set_priority(struct channel_gk20a *ch,
1785                 u32 priority)
1786 {
1787         u32 timeslice_timeout;
1788         /* set priority of graphics channel */
1789         switch (priority) {
1790         case NVHOST_PRIORITY_LOW:
1791                 /* 64 << 3 = 512us */
1792                 timeslice_timeout = 64;
1793                 break;
1794         case NVHOST_PRIORITY_MEDIUM:
1795                 /* 128 << 3 = 1024us */
1796                 timeslice_timeout = 128;
1797                 break;
1798         case NVHOST_PRIORITY_HIGH:
1799                 /* 255 << 3 = 2048us */
1800                 timeslice_timeout = 255;
1801                 break;
1802         default:
1803                 pr_err("Unsupported priority");
1804                 return -EINVAL;
1805         }
1806         channel_gk20a_set_schedule_params(ch,
1807                         timeslice_timeout);
1808         return 0;
1809 }
1810
1811 static int gk20a_channel_zcull_bind(struct channel_gk20a *ch,
1812                             struct nvhost_zcull_bind_args *args)
1813 {
1814         struct gk20a *g = ch->g;
1815         struct gr_gk20a *gr = &g->gr;
1816
1817         gk20a_dbg_fn("");
1818
1819         return gr_gk20a_bind_ctxsw_zcull(g, gr, ch,
1820                                 args->gpu_va, args->mode);
1821 }
1822
1823 /* in this context the "channel" is the host1x channel which
1824  * maps to *all* gk20a channels */
1825 int gk20a_channel_suspend(struct gk20a *g)
1826 {
1827         struct fifo_gk20a *f = &g->fifo;
1828         u32 chid;
1829         bool channels_in_use = false;
1830         int err;
1831
1832         gk20a_dbg_fn("");
1833
1834         /* wait for engine idle */
1835         err = gk20a_fifo_wait_engine_idle(g);
1836         if (err)
1837                 return err;
1838
1839         for (chid = 0; chid < f->num_channels; chid++) {
1840                 if (f->channel[chid].in_use) {
1841
1842                         gk20a_dbg_info("suspend channel %d", chid);
1843                         /* disable channel */
1844                         gk20a_writel(g, ccsr_channel_r(chid),
1845                                 gk20a_readl(g, ccsr_channel_r(chid)) |
1846                                 ccsr_channel_enable_clr_true_f());
1847                         /* preempt the channel */
1848                         gk20a_fifo_preempt_channel(g, chid);
1849
1850                         channels_in_use = true;
1851                 }
1852         }
1853
1854         if (channels_in_use) {
1855                 gk20a_fifo_update_runlist(g, 0, ~0, false, true);
1856
1857                 for (chid = 0; chid < f->num_channels; chid++) {
1858                         if (f->channel[chid].in_use)
1859                                 channel_gk20a_unbind(&f->channel[chid]);
1860                 }
1861         }
1862
1863         gk20a_dbg_fn("done");
1864         return 0;
1865 }
1866
1867 /* in this context the "channel" is the host1x channel which
1868  * maps to *all* gk20a channels */
1869 int gk20a_channel_resume(struct gk20a *g)
1870 {
1871         struct fifo_gk20a *f = &g->fifo;
1872         u32 chid;
1873         bool channels_in_use = false;
1874
1875         gk20a_dbg_fn("");
1876
1877         for (chid = 0; chid < f->num_channels; chid++) {
1878                 if (f->channel[chid].in_use) {
1879                         gk20a_dbg_info("resume channel %d", chid);
1880                         g->ops.fifo.bind_channel(&f->channel[chid]);
1881                         channels_in_use = true;
1882                 }
1883         }
1884
1885         if (channels_in_use)
1886                 gk20a_fifo_update_runlist(g, 0, ~0, true, true);
1887
1888         gk20a_dbg_fn("done");
1889         return 0;
1890 }
1891
1892 void gk20a_channel_semaphore_wakeup(struct gk20a *g)
1893 {
1894         struct fifo_gk20a *f = &g->fifo;
1895         u32 chid;
1896
1897         gk20a_dbg_fn("");
1898
1899         for (chid = 0; chid < f->num_channels; chid++) {
1900                 struct channel_gk20a *c = g->fifo.channel+chid;
1901                 if (c->in_use)
1902                         wake_up_interruptible_all(&c->semaphore_wq);
1903         }
1904 }
1905
1906 static int gk20a_ioctl_channel_submit_gpfifo(
1907         struct channel_gk20a *ch,
1908         struct nvhost_submit_gpfifo_args *args)
1909 {
1910         void *gpfifo;
1911         u32 size;
1912         int ret = 0;
1913
1914         gk20a_dbg_fn("");
1915
1916         if (ch->has_timedout)
1917                 return -ETIMEDOUT;
1918
1919         size = args->num_entries * sizeof(struct nvhost_gpfifo);
1920
1921         gpfifo = kzalloc(size, GFP_KERNEL);
1922         if (!gpfifo)
1923                 return -ENOMEM;
1924
1925         if (copy_from_user(gpfifo,
1926                            (void __user *)(uintptr_t)args->gpfifo, size)) {
1927                 ret = -EINVAL;
1928                 goto clean_up;
1929         }
1930
1931         ret = gk20a_submit_channel_gpfifo(ch, gpfifo, args->num_entries,
1932                                         &args->fence, args->flags);
1933
1934 clean_up:
1935         kfree(gpfifo);
1936         return ret;
1937 }
1938
1939 void gk20a_init_channel(struct gpu_ops *gops)
1940 {
1941         gops->fifo.bind_channel = channel_gk20a_bind;
1942 }
1943
1944 long gk20a_channel_ioctl(struct file *filp,
1945         unsigned int cmd, unsigned long arg)
1946 {
1947         struct channel_gk20a *ch = filp->private_data;
1948         struct platform_device *dev = ch->g->dev;
1949         u8 buf[NVHOST_IOCTL_CHANNEL_MAX_ARG_SIZE];
1950         int err = 0;
1951
1952         if ((_IOC_TYPE(cmd) != NVHOST_IOCTL_MAGIC) ||
1953                 (_IOC_NR(cmd) == 0) ||
1954                 (_IOC_NR(cmd) > NVHOST_IOCTL_CHANNEL_LAST) ||
1955                 (_IOC_SIZE(cmd) > NVHOST_IOCTL_CHANNEL_MAX_ARG_SIZE))
1956                 return -EFAULT;
1957
1958         if (_IOC_DIR(cmd) & _IOC_WRITE) {
1959                 if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd)))
1960                         return -EFAULT;
1961         }
1962
1963         switch (cmd) {
1964         case NVHOST_IOCTL_CHANNEL_OPEN:
1965         {
1966                 int fd;
1967                 struct file *file;
1968                 char *name;
1969
1970                 err = get_unused_fd_flags(O_RDWR);
1971                 if (err < 0)
1972                         break;
1973                 fd = err;
1974
1975                 name = kasprintf(GFP_KERNEL, "nvhost-%s-fd%d",
1976                                 dev_name(&dev->dev), fd);
1977                 if (!name) {
1978                         err = -ENOMEM;
1979                         put_unused_fd(fd);
1980                         break;
1981                 }
1982
1983                 file = anon_inode_getfile(name, filp->f_op, NULL, O_RDWR);
1984                 kfree(name);
1985                 if (IS_ERR(file)) {
1986                         err = PTR_ERR(file);
1987                         put_unused_fd(fd);
1988                         break;
1989                 }
1990                 fd_install(fd, file);
1991
1992                 err = __gk20a_channel_open(ch->g, file);
1993                 if (err) {
1994                         put_unused_fd(fd);
1995                         fput(file);
1996                         break;
1997                 }
1998
1999                 ((struct nvhost_channel_open_args *)buf)->channel_fd = fd;
2000                 break;
2001         }
2002         case NVHOST_IOCTL_CHANNEL_SET_NVMAP_FD:
2003                 break;
2004         case NVHOST_IOCTL_CHANNEL_ALLOC_OBJ_CTX:
2005                 err = gk20a_busy(dev);
2006                 if (err) {
2007                         dev_err(&dev->dev,
2008                                 "%s: failed to host gk20a for ioctl cmd: 0x%x",
2009                                 __func__, cmd);
2010                         return err;
2011                 }
2012                 err = gk20a_alloc_obj_ctx(ch,
2013                                 (struct nvhost_alloc_obj_ctx_args *)buf);
2014                 gk20a_idle(dev);
2015                 break;
2016         case NVHOST_IOCTL_CHANNEL_FREE_OBJ_CTX:
2017                 err = gk20a_busy(dev);
2018                 if (err) {
2019                         dev_err(&dev->dev,
2020                                 "%s: failed to host gk20a for ioctl cmd: 0x%x",
2021                                 __func__, cmd);
2022                         return err;
2023                 }
2024                 err = gk20a_free_obj_ctx(ch,
2025                                 (struct nvhost_free_obj_ctx_args *)buf);
2026                 gk20a_idle(dev);
2027                 break;
2028         case NVHOST_IOCTL_CHANNEL_ALLOC_GPFIFO:
2029                 err = gk20a_busy(dev);
2030                 if (err) {
2031                         dev_err(&dev->dev,
2032                                 "%s: failed to host gk20a for ioctl cmd: 0x%x",
2033                                 __func__, cmd);
2034                         return err;
2035                 }
2036                 err = gk20a_alloc_channel_gpfifo(ch,
2037                                 (struct nvhost_alloc_gpfifo_args *)buf);
2038                 gk20a_idle(dev);
2039                 break;
2040         case NVHOST_IOCTL_CHANNEL_SUBMIT_GPFIFO:
2041                 err = gk20a_ioctl_channel_submit_gpfifo(ch,
2042                                 (struct nvhost_submit_gpfifo_args *)buf);
2043                 break;
2044         case NVHOST_IOCTL_CHANNEL_WAIT:
2045                 err = gk20a_busy(dev);
2046                 if (err) {
2047                         dev_err(&dev->dev,
2048                                 "%s: failed to host gk20a for ioctl cmd: 0x%x",
2049                                 __func__, cmd);
2050                         return err;
2051                 }
2052                 err = gk20a_channel_wait(ch,
2053                                 (struct nvhost_wait_args *)buf);
2054                 gk20a_idle(dev);
2055                 break;
2056         case NVHOST_IOCTL_CHANNEL_ZCULL_BIND:
2057                 err = gk20a_busy(dev);
2058                 if (err) {
2059                         dev_err(&dev->dev,
2060                                 "%s: failed to host gk20a for ioctl cmd: 0x%x",
2061                                 __func__, cmd);
2062                         return err;
2063                 }
2064                 err = gk20a_channel_zcull_bind(ch,
2065                                 (struct nvhost_zcull_bind_args *)buf);
2066                 gk20a_idle(dev);
2067                 break;
2068         case NVHOST_IOCTL_CHANNEL_SET_ERROR_NOTIFIER:
2069                 err = gk20a_busy(dev);
2070                 if (err) {
2071                         dev_err(&dev->dev,
2072                                 "%s: failed to host gk20a for ioctl cmd: 0x%x",
2073                                 __func__, cmd);
2074                         return err;
2075                 }
2076                 err = gk20a_init_error_notifier(ch,
2077                                 (struct nvhost_set_error_notifier *)buf);
2078                 gk20a_idle(dev);
2079                 break;
2080 #ifdef CONFIG_GK20A_CYCLE_STATS
2081         case NVHOST_IOCTL_CHANNEL_CYCLE_STATS:
2082                 err = gk20a_busy(dev);
2083                 if (err) {
2084                         dev_err(&dev->dev,
2085                                 "%s: failed to host gk20a for ioctl cmd: 0x%x",
2086                                 __func__, cmd);
2087                         return err;
2088                 }
2089                 err = gk20a_channel_cycle_stats(ch,
2090                                 (struct nvhost_cycle_stats_args *)buf);
2091                 gk20a_idle(dev);
2092                 break;
2093 #endif
2094         case NVHOST_IOCTL_CHANNEL_SET_TIMEOUT:
2095         {
2096                 u32 timeout =
2097                         (u32)((struct nvhost_set_timeout_args *)buf)->timeout;
2098                 gk20a_dbg(gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d",
2099                            timeout, ch->hw_chid);
2100                 ch->timeout_ms_max = timeout;
2101                 break;
2102         }
2103         case NVHOST_IOCTL_CHANNEL_SET_TIMEOUT_EX:
2104         {
2105                 u32 timeout =
2106                         (u32)((struct nvhost_set_timeout_args *)buf)->timeout;
2107                 bool timeout_debug_dump = !((u32)
2108                         ((struct nvhost_set_timeout_ex_args *)buf)->flags &
2109                         (1 << NVHOST_TIMEOUT_FLAG_DISABLE_DUMP));
2110                 gk20a_dbg(gpu_dbg_gpu_dbg, "setting timeout (%d ms) for chid %d",
2111                            timeout, ch->hw_chid);
2112                 ch->timeout_ms_max = timeout;
2113                 ch->timeout_debug_dump = timeout_debug_dump;
2114                 break;
2115         }
2116         case NVHOST_IOCTL_CHANNEL_GET_TIMEDOUT:
2117                 ((struct nvhost_get_param_args *)buf)->value =
2118                         ch->has_timedout;
2119                 break;
2120         case NVHOST_IOCTL_CHANNEL_SET_PRIORITY:
2121                 err = gk20a_busy(dev);
2122                 if (err) {
2123                         dev_err(&dev->dev,
2124                                 "%s: failed to host gk20a for ioctl cmd: 0x%x",
2125                                 __func__, cmd);
2126                         return err;
2127                 }
2128                 gk20a_channel_set_priority(ch,
2129                         ((struct nvhost_set_priority_args *)buf)->priority);
2130                 gk20a_idle(dev);
2131                 break;
2132         default:
2133                 dev_err(&dev->dev, "unrecognized ioctl cmd: 0x%x", cmd);
2134                 err = -ENOTTY;
2135                 break;
2136         }
2137
2138         if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
2139                 err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd));
2140
2141         return err;
2142 }