drivers/video/tegra/host/host1x/host1x_channel.c

   1 /*
   2  * drivers/video/tegra/host/host1x/channel_host1x.c
   3  *
   4  * Tegra Graphics Host Channel
   5  *
   6  * Copyright (c) 2010-2014, NVIDIA CORPORATION.  All rights reserved.
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms and conditions of the GNU General Public License,
  10  * version 2, as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope it will be useful, but WITHOUT
  13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15  * more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #include "nvhost_channel.h"
  22 #include "dev.h"
  23 #include "class_ids.h"
  24 #include "nvhost_acm.h"
  25 #include "nvhost_job.h"
  26 #include "nvhost_hwctx.h"
  27 #include <trace/events/nvhost.h>
  28 #include <linux/slab.h>
  29 #include "nvhost_sync.h"
  30
  31 #include "nvhost_hwctx.h"
  32 #include "nvhost_intr.h"
  33 #include "class_ids.h"
  34
  35 static void sync_waitbases(struct nvhost_channel *ch, u32 syncpt_val)
  36 {
  37         unsigned long waitbase;
  38         struct nvhost_device_data *pdata = platform_get_drvdata(ch->dev);
  39         if (pdata->waitbasesync) {
  40                 waitbase = pdata->waitbases[0];
  41                 nvhost_cdma_push(&ch->cdma,
  42                         nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
  43                                 host1x_uclass_load_syncpt_base_r(),
  44                                 1),
  45                                 nvhost_class_host_load_syncpt_base(waitbase,
  46                                                 syncpt_val));
  47         }
  48 }
  49
  50 static void serialize(struct nvhost_job *job)
  51 {
  52         struct nvhost_channel *ch = job->ch;
  53         struct nvhost_syncpt *sp = &nvhost_get_host(ch->dev)->syncpt;
  54         struct nvhost_device_data *pdata = platform_get_drvdata(ch->dev);
  55         int i;
  56
  57         if (!job->serialize && !pdata->serialize)
  58                 return;
  59
  60         /*
  61          * Force serialization by inserting a host wait for the
  62          * previous job to finish before this one can commence.
  63          *
  64          * NOTE! This cannot be packed because otherwise we might
  65          * overwrite the RESTART opcode at the end of the push
  66          * buffer.
  67          */
  68
  69         for (i = 0; i < job->num_syncpts; ++i) {
  70                 u32 id = job->sp[i].id;
  71                 u32 max = nvhost_syncpt_read_max(sp, id);
  72
  73                 nvhost_cdma_push(&ch->cdma,
  74                         nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
  75                                 host1x_uclass_wait_syncpt_r(), 1),
  76                         nvhost_class_host_wait_syncpt(id, max));
  77         }
  78 }
  79
  80 static bool ctxsave_needed(struct nvhost_job *job, struct nvhost_hwctx *cur_ctx)
  81 {
  82         struct nvhost_channel *ch = job->ch;
  83
  84         if (!cur_ctx || ch->cur_ctx == job->hwctx ||
  85                         ch->cur_ctx->has_timedout ||
  86                         !ch->cur_ctx->h->save_push)
  87                 return false;
  88         else
  89                 return true;
  90 }
  91
  92 static void submit_ctxsave(struct nvhost_job *job, struct nvhost_hwctx *cur_ctx)
  93 {
  94         struct nvhost_master *host = nvhost_get_host(job->ch->dev);
  95         struct nvhost_channel *ch = job->ch;
  96         u32 syncval;
  97
  98         /* Is a save needed? */
  99         if (!ctxsave_needed(job, cur_ctx))
 100                 return;
 101
 102         /* Adjust the syncpoint max */
 103         job->sp[job->hwctx_syncpt_idx].incrs +=
 104                 cur_ctx->save_incrs;
 105         syncval = nvhost_syncpt_incr_max(&host->syncpt,
 106                         job->sp[job->hwctx_syncpt_idx].id,
 107                         cur_ctx->save_incrs);
 108
 109         /* Send the save to channel */
 110         cur_ctx->valid = true;
 111         cur_ctx->h->save_push(cur_ctx, &ch->cdma);
 112         nvhost_job_get_hwctx(job, cur_ctx);
 113
 114         trace_nvhost_channel_context_save(ch->dev->name, cur_ctx);
 115 }
 116
 117 static void add_sync_waits(struct nvhost_channel *ch, int fd)
 118 {
 119         struct nvhost_master *host = nvhost_get_host(ch->dev);
 120         struct nvhost_syncpt *sp = &host->syncpt;
 121         struct sync_fence *fence;
 122         struct sync_pt *_pt;
 123         struct nvhost_sync_pt *pt;
 124         struct list_head *pos;
 125
 126         if (fd < 0)
 127                 return;
 128
 129         fence = nvhost_sync_fdget(fd);
 130         if (!fence)
 131                 return;
 132
 133         /*
 134          * Force serialization by inserting a host wait for the
 135          * previous job to finish before this one can commence.
 136          *
 137          * NOTE! This cannot be packed because otherwise we might
 138          * overwrite the RESTART opcode at the end of the push
 139          * buffer.
 140          */
 141
 142         list_for_each(pos, &fence->pt_list_head) {
 143                 u32 id;
 144                 u32 thresh;
 145
 146                 _pt = container_of(pos, struct sync_pt, pt_list);
 147                 pt = to_nvhost_sync_pt(_pt);
 148                 id = nvhost_sync_pt_id(pt);
 149                 thresh = nvhost_sync_pt_thresh(pt);
 150
 151                 if (nvhost_syncpt_is_expired(sp, id, thresh))
 152                         continue;
 153
 154                 nvhost_cdma_push(&ch->cdma,
 155                         nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
 156                                 host1x_uclass_wait_syncpt_r(), 1),
 157                         nvhost_class_host_wait_syncpt(id, thresh));
 158         }
 159         sync_fence_put(fence);
 160 }
 161
 162 static void submit_ctxrestore(struct nvhost_job *job)
 163 {
 164         struct nvhost_master *host = nvhost_get_host(job->ch->dev);
 165         struct nvhost_channel *ch = job->ch;
 166         u32 syncval;
 167         struct nvhost_hwctx *ctx = job->hwctx;
 168
 169         /* First check if we have a valid context to restore */
 170         if (ch->cur_ctx == job->hwctx || !job->hwctx ||
 171                 !job->hwctx->valid ||
 172                 !ctx->h->restore_push)
 173                 return;
 174
 175         /* Increment syncpt max */
 176         job->sp[job->hwctx_syncpt_idx].incrs += ctx->restore_incrs;
 177         syncval = nvhost_syncpt_incr_max(&host->syncpt,
 178                         job->sp[job->hwctx_syncpt_idx].id,
 179                         ctx->restore_incrs);
 180
 181         /* Send restore buffer to channel */
 182         ctx->h->restore_push(ctx, &ch->cdma);
 183
 184         trace_nvhost_channel_context_restore(ch->dev->name, ctx);
 185 }
 186
 187 static void submit_nullkickoff(struct nvhost_job *job, u32 user_syncpt_incrs)
 188 {
 189         struct nvhost_channel *ch = job->ch;
 190         int incr, i;
 191         u32 op_incr;
 192         struct nvhost_device_data *pdata = platform_get_drvdata(ch->dev);
 193
 194         /* push increments that correspond to nulled out commands */
 195         for (i = 0; i < job->num_syncpts; ++i) {
 196                 u32 incrs = (i == job->hwctx_syncpt_idx) ?
 197                         user_syncpt_incrs : job->sp[i].incrs;
 198                 op_incr = nvhost_opcode_imm_incr_syncpt(
 199                         host1x_uclass_incr_syncpt_cond_op_done_v(),
 200                         job->sp[i].id);
 201                 for (incr = 0; incr < (incrs >> 1); incr++)
 202                         nvhost_cdma_push(&ch->cdma, op_incr, op_incr);
 203                 if (incrs & 1)
 204                         nvhost_cdma_push(&ch->cdma, op_incr,
 205                                 NVHOST_OPCODE_NOOP);
 206         }
 207
 208         /* for 3d, waitbase needs to be incremented after each submit */
 209         if (pdata->class == NV_GRAPHICS_3D_CLASS_ID) {
 210                 u32 waitbase = job->hwctx->h->waitbase;
 211                 nvhost_cdma_push(&ch->cdma,
 212                         nvhost_opcode_setclass(
 213                                 NV_HOST1X_CLASS_ID,
 214                                 host1x_uclass_incr_syncpt_base_r(),
 215                                 1),
 216                         nvhost_class_host_incr_syncpt_base(
 217                                 waitbase,
 218                                 job->sp[job->hwctx_syncpt_idx].incrs));
 219         }
 220 }
 221
 222 static inline u32 gather_regnum(u32 word)
 223 {
 224         return (word >> 16) & 0xfff;
 225 }
 226
 227 static inline  u32 gather_type(u32 word)
 228 {
 229         return (word >> 28) & 1;
 230 }
 231
 232 static inline u32 gather_count(u32 word)
 233 {
 234         return word & 0x3fff;
 235 }
 236
 237 static void submit_gathers(struct nvhost_job *job)
 238 {
 239         u32 class_id = 0;
 240         int i;
 241         void *cpuva;
 242
 243         /* push user gathers */
 244         for (i = 0 ; i < job->num_gathers; i++) {
 245                 struct nvhost_job_gather *g = &job->gathers[i];
 246                 u32 op1;
 247                 u32 op2;
 248
 249                 add_sync_waits(job->ch, g->pre_fence);
 250
 251                 if (g->class_id != class_id) {
 252                         nvhost_cdma_push(&job->ch->cdma,
 253                                 nvhost_opcode_setclass(g->class_id, 0, 0),
 254                                 NVHOST_OPCODE_NOOP);
 255                         class_id = g->class_id;
 256                 }
 257
 258                 /* If register is specified, add a gather with incr/nonincr.
 259                  * This allows writing large amounts of data directly from
 260                  * memory to a register. */
 261                 if (gather_regnum(g->words))
 262                         op1 = nvhost_opcode_gather_insert(
 263                                         gather_regnum(g->words),
 264                                         gather_type(g->words),
 265                                         gather_count(g->words));
 266                 else
 267                         op1 = nvhost_opcode_gather(g->words);
 268                 op2 = job->gathers[i].mem_base + g->offset;
 269
 270                 cpuva = dma_buf_vmap(g->buf);
 271                 nvhost_cdma_push_gather(&job->ch->cdma,
 272                                 cpuva,
 273                                 job->gathers[i].mem_base,
 274                                 g->offset,
 275                                 op1, op2);
 276                 dma_buf_vunmap(g->buf, cpuva);
 277         }
 278 }
 279
 280 static int host1x_channel_submit(struct nvhost_job *job)
 281 {
 282         struct nvhost_channel *ch = job->ch;
 283         struct nvhost_syncpt *sp = &nvhost_get_host(job->ch->dev)->syncpt;
 284         u32 user_syncpt_incrs;
 285         u32 prev_max = 0;
 286         int err, i;
 287         void *completed_waiters[job->num_syncpts];
 288         struct nvhost_job_syncpt *hwctx_sp = job->sp + job->hwctx_syncpt_idx;
 289
 290         memset(completed_waiters, 0, sizeof(void *) * job->num_syncpts);
 291
 292         /* Bail out on timed out contexts */
 293         if (job->hwctx && job->hwctx->has_timedout)
 294                 return -ETIMEDOUT;
 295
 296         /* Turn on the client module and host1x */
 297         for (i = 0; i < job->num_syncpts; ++i)
 298                 nvhost_module_busy(ch->dev);
 299
 300         /* before error checks, return current max */
 301         prev_max = hwctx_sp->fence = nvhost_syncpt_read_max(sp, hwctx_sp->id);
 302
 303         /* get submit lock */
 304         err = mutex_lock_interruptible(&ch->submitlock);
 305         if (err) {
 306                 nvhost_module_idle_mult(ch->dev, job->num_syncpts);
 307                 goto error;
 308         }
 309
 310         for (i = 0; i < job->num_syncpts; ++i) {
 311                 completed_waiters[i] = nvhost_intr_alloc_waiter();
 312                 if (!completed_waiters[i]) {
 313                         nvhost_module_idle_mult(ch->dev, job->num_syncpts);
 314                         mutex_unlock(&ch->submitlock);
 315                         err = -ENOMEM;
 316                         goto error;
 317                 }
 318                 if (nvhost_intr_has_pending_jobs(
 319                         &nvhost_get_host(ch->dev)->intr, job->sp[i].id, ch))
 320                         dev_warn(&ch->dev->dev,
 321                                 "%s: cross-channel dependencies on syncpt %d\n",
 322                                 __func__, job->sp[i].id);
 323         }
 324
 325         /* begin a CDMA submit */
 326         err = nvhost_cdma_begin(&ch->cdma, job);
 327         if (err) {
 328                 mutex_unlock(&ch->submitlock);
 329                 nvhost_module_idle_mult(ch->dev, job->num_syncpts);
 330                 goto error;
 331         }
 332
 333         serialize(job);
 334
 335         /* submit_ctxsave() and submit_ctxrestore() use the channel syncpt */
 336         user_syncpt_incrs = hwctx_sp->incrs;
 337
 338         submit_ctxsave(job, ch->cur_ctx);
 339         submit_ctxrestore(job);
 340         ch->cur_ctx = job->hwctx;
 341
 342         /* determine fences for all syncpoints */
 343         for (i = 0; i < job->num_syncpts; ++i) {
 344                 u32 incrs = (i == job->hwctx_syncpt_idx) ?
 345                         user_syncpt_incrs :
 346                         job->sp[i].incrs;
 347
 348                 /* create a valid max for client managed syncpoints */
 349                 if (nvhost_syncpt_client_managed(sp, job->sp[i].id)) {
 350                         u32 min = nvhost_syncpt_read(sp, job->sp[i].id);
 351                         if (min)
 352                                 dev_warn(&job->ch->dev->dev,
 353                                         "converting an active unmanaged syncpoint %d to managed\n",
 354                                         job->sp[i].id);
 355                         nvhost_syncpt_set_max(sp, job->sp[i].id, min);
 356                         nvhost_syncpt_set_manager(sp, job->sp[i].id, false);
 357                 }
 358
 359                 job->sp[i].fence =
 360                         nvhost_syncpt_incr_max(sp, job->sp[i].id, incrs);
 361         }
 362
 363         if (job->null_kickoff)
 364                 submit_nullkickoff(job, user_syncpt_incrs);
 365         else
 366                 submit_gathers(job);
 367
 368         sync_waitbases(ch, hwctx_sp->fence);
 369
 370         /* end CDMA submit & stash pinned hMems into sync queue */
 371         nvhost_cdma_end(&ch->cdma, job);
 372
 373         trace_nvhost_channel_submitted(ch->dev->name, prev_max,
 374                 hwctx_sp->fence);
 375
 376         for (i = 0; i < job->num_syncpts; ++i) {
 377                 /* schedule a submit complete interrupt */
 378                 err = nvhost_intr_add_action(&nvhost_get_host(ch->dev)->intr,
 379                         job->sp[i].id, job->sp[i].fence,
 380                         NVHOST_INTR_ACTION_SUBMIT_COMPLETE, ch,
 381                         completed_waiters[i],
 382                         NULL);
 383                 WARN(err, "Failed to set submit complete interrupt");
 384         }
 385
 386         mutex_unlock(&ch->submitlock);
 387
 388         return 0;
 389
 390 error:
 391         for (i = 0; i < job->num_syncpts; ++i)
 392                 kfree(completed_waiters[i]);
 393
 394         return err;
 395 }
 396
 397 static int host1x_save_context(struct nvhost_channel *ch)
 398 {
 399         struct nvhost_hwctx *hwctx_to_save;
 400         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
 401         u32 syncpt_incrs, syncpt_val;
 402         int err = 0;
 403         void *ref;
 404         void *wakeup_waiter = NULL;
 405         struct nvhost_job *job;
 406         u32 syncpt_id, waitbase;
 407
 408         wakeup_waiter = nvhost_intr_alloc_waiter();
 409         if (!wakeup_waiter) {
 410                 err = -ENOMEM;
 411                 goto done;
 412         }
 413
 414         nvhost_module_busy(nvhost_get_parent(ch->dev));
 415
 416         mutex_lock(&ch->submitlock);
 417         hwctx_to_save = ch->cur_ctx;
 418         if (!hwctx_to_save) {
 419                 mutex_unlock(&ch->submitlock);
 420                 goto done;
 421         }
 422
 423         job = nvhost_job_alloc(ch, hwctx_to_save, 0, 0, 0, 1);
 424         if (!job) {
 425                 err = -ENOMEM;
 426                 mutex_unlock(&ch->submitlock);
 427                 goto done;
 428         }
 429
 430         hwctx_to_save->valid = true;
 431         ch->cur_ctx = NULL;
 432         syncpt_id = hwctx_to_save->h->syncpt;
 433         waitbase = hwctx_to_save->h->waitbase;
 434
 435         syncpt_incrs = hwctx_to_save->save_incrs;
 436         syncpt_val = nvhost_syncpt_incr_max(&nvhost_get_host(ch->dev)->syncpt,
 437                                         syncpt_id, syncpt_incrs);
 438
 439         job->hwctx_syncpt_idx = 0;
 440         job->sp->id = syncpt_id;
 441         job->sp->waitbase = waitbase;
 442         job->sp->incrs = syncpt_incrs;
 443         job->sp->fence = syncpt_val;
 444         job->num_syncpts = 1;
 445
 446         err = nvhost_cdma_begin(&ch->cdma, job);
 447         if (err) {
 448                 mutex_unlock(&ch->submitlock);
 449                 goto done;
 450         }
 451
 452         hwctx_to_save->h->save_push(hwctx_to_save, &ch->cdma);
 453         nvhost_cdma_end(&ch->cdma, job);
 454         nvhost_job_put(job);
 455         job = NULL;
 456
 457         err = nvhost_intr_add_action(&nvhost_get_host(ch->dev)->intr,
 458                         syncpt_id, syncpt_val,
 459                         NVHOST_INTR_ACTION_WAKEUP, &wq,
 460                         wakeup_waiter,
 461                         &ref);
 462         wakeup_waiter = NULL;
 463         WARN(err, "Failed to set wakeup interrupt");
 464         wait_event(wq,
 465                 nvhost_syncpt_is_expired(&nvhost_get_host(ch->dev)->syncpt,
 466                                 syncpt_id, syncpt_val));
 467
 468         nvhost_intr_put_ref(&nvhost_get_host(ch->dev)->intr, syncpt_id, ref);
 469
 470         nvhost_cdma_update(&ch->cdma);
 471
 472         mutex_unlock(&ch->submitlock);
 473         nvhost_module_idle(nvhost_get_parent(ch->dev));
 474
 475 done:
 476         kfree(wakeup_waiter);
 477         return err;
 478 }
 479
 480 static inline void __iomem *host1x_channel_aperture(void __iomem *p, int ndx)
 481 {
 482         p += ndx * NV_HOST1X_CHANNEL_MAP_SIZE_BYTES;
 483         return p;
 484 }
 485
 486 static inline int hwctx_handler_init(struct nvhost_channel *ch)
 487 {
 488         int err = 0;
 489
 490         struct nvhost_device_data *pdata = platform_get_drvdata(ch->dev);
 491         u32 syncpt = pdata->syncpts[0];
 492         u32 waitbase = pdata->waitbases[0];
 493
 494         if (pdata->alloc_hwctx_handler) {
 495                 ch->ctxhandler = pdata->alloc_hwctx_handler(syncpt,
 496                                 waitbase, ch);
 497                 if (!ch->ctxhandler)
 498                         err = -ENOMEM;
 499         }
 500
 501         return err;
 502 }
 503
 504 static int host1x_channel_init(struct nvhost_channel *ch,
 505         struct nvhost_master *dev)
 506 {
 507         mutex_init(&ch->reflock);
 508         mutex_init(&ch->submitlock);
 509
 510         ch->aperture = host1x_channel_aperture(dev->aperture, ch->chid);
 511
 512         return hwctx_handler_init(ch);
 513 }
 514
 515 static const struct nvhost_channel_ops host1x_channel_ops = {
 516         .init = host1x_channel_init,
 517         .submit = host1x_channel_submit,
 518         .save_context = host1x_save_context,
 519 };