drivers/usb/dwc3/gadget.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   4  *
   5  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
   6  *
   7  * Authors: Felipe Balbi <balbi@ti.com>,
   8  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/delay.h>
  13 #include <linux/slab.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/pm_runtime.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/io.h>
  19 #include <linux/list.h>
  20 #include <linux/dma-mapping.h>
  21
  22 #include <linux/usb/ch9.h>
  23 #include <linux/usb/gadget.h>
  24
  25 #include "debug.h"
  26 #include "core.h"
  27 #include "gadget.h"
  28 #include "io.h"
  29
  30 #define DWC3_ALIGN_FRAME(d)     (((d)->frame_number + (d)->interval) \
  31                                         & ~((d)->interval - 1))
  32
  33 /**
  34  * dwc3_gadget_set_test_mode - enables usb2 test modes
  35  * @dwc: pointer to our context structure
  36  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
  37  *
  38  * Caller should take care of locking. This function will return 0 on
  39  * success or -EINVAL if wrong Test Selector is passed.
  40  */
  41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
  42 {
  43         u32             reg;
  44
  45         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
  46         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
  47
  48         switch (mode) {
  49         case TEST_J:
  50         case TEST_K:
  51         case TEST_SE0_NAK:
  52         case TEST_PACKET:
  53         case TEST_FORCE_EN:
  54                 reg |= mode << 1;
  55                 break;
  56         default:
  57                 return -EINVAL;
  58         }
  59
  60         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
  61
  62         return 0;
  63 }
  64
  65 /**
  66  * dwc3_gadget_get_link_state - gets current state of usb link
  67  * @dwc: pointer to our context structure
  68  *
  69  * Caller should take care of locking. This function will
  70  * return the link state on success (>= 0) or -ETIMEDOUT.
  71  */
  72 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
  73 {
  74         u32             reg;
  75
  76         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  77
  78         return DWC3_DSTS_USBLNKST(reg);
  79 }
  80
  81 /**
  82  * dwc3_gadget_set_link_state - sets usb link to a particular state
  83  * @dwc: pointer to our context structure
  84  * @state: the state to put link into
  85  *
  86  * Caller should take care of locking. This function will
  87  * return 0 on success or -ETIMEDOUT.
  88  */
  89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
  90 {
  91         int             retries = 10000;
  92         u32             reg;
  93
  94         /*
  95          * Wait until device controller is ready. Only applies to 1.94a and
  96          * later RTL.
  97          */
  98         if (dwc->revision >= DWC3_REVISION_194A) {
  99                 while (--retries) {
 100                         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 101                         if (reg & DWC3_DSTS_DCNRD)
 102                                 udelay(5);
 103                         else
 104                                 break;
 105                 }
 106
 107                 if (retries <= 0)
 108                         return -ETIMEDOUT;
 109         }
 110
 111         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 112         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
 113
 114         /* set requested state */
 115         reg |= DWC3_DCTL_ULSTCHNGREQ(state);
 116         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 117
 118         /*
 119          * The following code is racy when called from dwc3_gadget_wakeup,
 120          * and is not needed, at least on newer versions
 121          */
 122         if (dwc->revision >= DWC3_REVISION_194A)
 123                 return 0;
 124
 125         /* wait for a change in DSTS */
 126         retries = 10000;
 127         while (--retries) {
 128                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 129
 130                 if (DWC3_DSTS_USBLNKST(reg) == state)
 131                         return 0;
 132
 133                 udelay(5);
 134         }
 135
 136         return -ETIMEDOUT;
 137 }
 138
 139 /**
 140  * dwc3_ep_inc_trb - increment a trb index.
 141  * @index: Pointer to the TRB index to increment.
 142  *
 143  * The index should never point to the link TRB. After incrementing,
 144  * if it is point to the link TRB, wrap around to the beginning. The
 145  * link TRB is always at the last TRB entry.
 146  */
 147 static void dwc3_ep_inc_trb(u8 *index)
 148 {
 149         (*index)++;
 150         if (*index == (DWC3_TRB_NUM - 1))
 151                 *index = 0;
 152 }
 153
 154 /**
 155  * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
 156  * @dep: The endpoint whose enqueue pointer we're incrementing
 157  */
 158 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
 159 {
 160         dwc3_ep_inc_trb(&dep->trb_enqueue);
 161 }
 162
 163 /**
 164  * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
 165  * @dep: The endpoint whose enqueue pointer we're incrementing
 166  */
 167 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
 168 {
 169         dwc3_ep_inc_trb(&dep->trb_dequeue);
 170 }
 171
 172 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
 173                 struct dwc3_request *req, int status)
 174 {
 175         struct dwc3                     *dwc = dep->dwc;
 176
 177         req->started = false;
 178         list_del(&req->list);
 179         req->remaining = 0;
 180
 181         if (req->request.status == -EINPROGRESS)
 182                 req->request.status = status;
 183
 184         if (req->trb)
 185                 usb_gadget_unmap_request_by_dev(dwc->sysdev,
 186                                 &req->request, req->direction);
 187
 188         req->trb = NULL;
 189         trace_dwc3_gadget_giveback(req);
 190
 191         if (dep->number > 1)
 192                 pm_runtime_put(dwc->dev);
 193 }
 194
 195 /**
 196  * dwc3_gadget_giveback - call struct usb_request's ->complete callback
 197  * @dep: The endpoint to whom the request belongs to
 198  * @req: The request we're giving back
 199  * @status: completion code for the request
 200  *
 201  * Must be called with controller's lock held and interrupts disabled. This
 202  * function will unmap @req and call its ->complete() callback to notify upper
 203  * layers that it has completed.
 204  */
 205 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 206                 int status)
 207 {
 208         struct dwc3                     *dwc = dep->dwc;
 209
 210         if (dep->stream_capable && timer_pending(&req->stream_timeout_timer))
 211                 del_timer(&req->stream_timeout_timer);
 212
 213         dwc3_gadget_del_and_unmap_request(dep, req, status);
 214
 215         spin_unlock(&dwc->lock);
 216         usb_gadget_giveback_request(&dep->endpoint, &req->request);
 217         spin_lock(&dwc->lock);
 218 }
 219
 220 /**
 221  * dwc3_send_gadget_generic_command - issue a generic command for the controller
 222  * @dwc: pointer to the controller context
 223  * @cmd: the command to be issued
 224  * @param: command parameter
 225  *
 226  * Caller should take care of locking. Issue @cmd with a given @param to @dwc
 227  * and wait for its completion.
 228  */
 229 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
 230 {
 231         u32             timeout = 500;
 232         int             status = 0;
 233         int             ret = 0;
 234         u32             reg;
 235
 236         dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
 237         dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
 238
 239         do {
 240                 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
 241                 if (!(reg & DWC3_DGCMD_CMDACT)) {
 242                         status = DWC3_DGCMD_STATUS(reg);
 243                         if (status)
 244                                 ret = -EINVAL;
 245                         break;
 246                 }
 247         } while (--timeout);
 248
 249         if (!timeout) {
 250                 ret = -ETIMEDOUT;
 251                 status = -ETIMEDOUT;
 252         }
 253
 254         trace_dwc3_gadget_generic_cmd(cmd, param, status);
 255
 256         return ret;
 257 }
 258
 259 static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
 260
 261 /**
 262  * dwc3_send_gadget_ep_cmd - issue an endpoint command
 263  * @dep: the endpoint to which the command is going to be issued
 264  * @cmd: the command to be issued
 265  * @params: parameters to the command
 266  *
 267  * Caller should handle locking. This function will issue @cmd with given
 268  * @params to @dep and wait for its completion.
 269  */
 270 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 271                 struct dwc3_gadget_ep_cmd_params *params)
 272 {
 273         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 274         struct dwc3             *dwc = dep->dwc;
 275         u32                     timeout = 1000;
 276         u32                     reg;
 277
 278         int                     cmd_status = 0;
 279         int                     susphy = false;
 280         int                     ret = -EINVAL;
 281
 282         /*
 283          * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if
 284          * we're issuing an endpoint command, we must check if
 285          * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it.
 286          *
 287          * We will also set SUSPHY bit to what it was before returning as stated
 288          * by the same section on Synopsys databook.
 289          */
 290         if (dwc->gadget.speed <= USB_SPEED_HIGH) {
 291                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 292                 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
 293                         susphy = true;
 294                         reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
 295                         dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 296                 }
 297         }
 298
 299         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
 300                 int             needs_wakeup;
 301
 302                 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
 303                                 dwc->link_state == DWC3_LINK_STATE_U2 ||
 304                                 dwc->link_state == DWC3_LINK_STATE_U3);
 305
 306                 if (unlikely(needs_wakeup)) {
 307                         ret = __dwc3_gadget_wakeup(dwc);
 308                         dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
 309                                         ret);
 310                 }
 311         }
 312
 313         dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
 314         dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
 315         dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
 316
 317         /*
 318          * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
 319          * not relying on XferNotReady, we can make use of a special "No
 320          * Response Update Transfer" command where we should clear both CmdAct
 321          * and CmdIOC bits.
 322          *
 323          * With this, we don't need to wait for command completion and can
 324          * straight away issue further commands to the endpoint.
 325          *
 326          * NOTICE: We're making an assumption that control endpoints will never
 327          * make use of Update Transfer command. This is a safe assumption
 328          * because we can never have more than one request at a time with
 329          * Control Endpoints. If anybody changes that assumption, this chunk
 330          * needs to be updated accordingly.
 331          */
 332         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
 333                         !usb_endpoint_xfer_isoc(desc))
 334                 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
 335         else
 336                 cmd |= DWC3_DEPCMD_CMDACT;
 337
 338         dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
 339         do {
 340                 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
 341                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 342                         cmd_status = DWC3_DEPCMD_STATUS(reg);
 343
 344                         switch (cmd_status) {
 345                         case 0:
 346                                 ret = 0;
 347                                 break;
 348                         case DEPEVT_TRANSFER_NO_RESOURCE:
 349                                 ret = -EINVAL;
 350                                 break;
 351                         case DEPEVT_TRANSFER_BUS_EXPIRY:
 352                                 /*
 353                                  * SW issues START TRANSFER command to
 354                                  * isochronous ep with future frame interval. If
 355                                  * future interval time has already passed when
 356                                  * core receives the command, it will respond
 357                                  * with an error status of 'Bus Expiry'.
 358                                  *
 359                                  * Instead of always returning -EINVAL, let's
 360                                  * give a hint to the gadget driver that this is
 361                                  * the case by returning -EAGAIN.
 362                                  */
 363                                 ret = -EAGAIN;
 364                                 break;
 365                         default:
 366                                 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
 367                         }
 368
 369                         break;
 370                 }
 371         } while (--timeout);
 372
 373         if (timeout == 0) {
 374                 ret = -ETIMEDOUT;
 375                 cmd_status = -ETIMEDOUT;
 376         }
 377
 378         trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
 379
 380         if (ret == 0) {
 381                 switch (DWC3_DEPCMD_CMD(cmd)) {
 382                 case DWC3_DEPCMD_STARTTRANSFER:
 383                         dep->flags |= DWC3_EP_TRANSFER_STARTED;
 384                         dwc3_gadget_ep_get_transfer_index(dep);
 385                         break;
 386                 case DWC3_DEPCMD_ENDTRANSFER:
 387                         dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
 388                         break;
 389                 default:
 390                         /* nothing */
 391                         break;
 392                 }
 393         }
 394
 395         if (unlikely(susphy)) {
 396                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 397                 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
 398                 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 399         }
 400
 401         return ret;
 402 }
 403
 404 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
 405 {
 406         struct dwc3 *dwc = dep->dwc;
 407         struct dwc3_gadget_ep_cmd_params params;
 408         u32 cmd = DWC3_DEPCMD_CLEARSTALL;
 409
 410         /*
 411          * As of core revision 2.60a the recommended programming model
 412          * is to set the ClearPendIN bit when issuing a Clear Stall EP
 413          * command for IN endpoints. This is to prevent an issue where
 414          * some (non-compliant) hosts may not send ACK TPs for pending
 415          * IN transfers due to a mishandled error condition. Synopsys
 416          * STAR 9000614252.
 417          */
 418         if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
 419             (dwc->gadget.speed >= USB_SPEED_SUPER))
 420                 cmd |= DWC3_DEPCMD_CLEARPENDIN;
 421
 422         memset(&params, 0, sizeof(params));
 423
 424         return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 425 }
 426
 427 dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep, struct dwc3_trb *trb)
 428 {
 429         u32             offset = (char *) trb - (char *) dep->trb_pool;
 430
 431         return dep->trb_pool_dma + offset;
 432 }
 433
 434 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 435 {
 436         struct dwc3             *dwc = dep->dwc;
 437
 438         if (dep->trb_pool)
 439                 return 0;
 440
 441         dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
 442                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 443                         &dep->trb_pool_dma, GFP_KERNEL);
 444         if (!dep->trb_pool) {
 445                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 446                                 dep->name);
 447                 return -ENOMEM;
 448         }
 449
 450         return 0;
 451 }
 452
 453 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 454 {
 455         struct dwc3             *dwc = dep->dwc;
 456
 457         dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 458                         dep->trb_pool, dep->trb_pool_dma);
 459
 460         dep->trb_pool = NULL;
 461         dep->trb_pool_dma = 0;
 462 }
 463
 464 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
 465 {
 466         struct dwc3_gadget_ep_cmd_params params;
 467
 468         memset(&params, 0x00, sizeof(params));
 469
 470         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 471
 472         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
 473                         &params);
 474 }
 475
 476 /**
 477  * dwc3_gadget_start_config - configure ep resources
 478  * @dep: endpoint that is being enabled
 479  *
 480  * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
 481  * completion, it will set Transfer Resource for all available endpoints.
 482  *
 483  * The assignment of transfer resources cannot perfectly follow the data book
 484  * due to the fact that the controller driver does not have all knowledge of the
 485  * configuration in advance. It is given this information piecemeal by the
 486  * composite gadget framework after every SET_CONFIGURATION and
 487  * SET_INTERFACE. Trying to follow the databook programming model in this
 488  * scenario can cause errors. For two reasons:
 489  *
 490  * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
 491  * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
 492  * incorrect in the scenario of multiple interfaces.
 493  *
 494  * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
 495  * endpoint on alt setting (8.1.6).
 496  *
 497  * The following simplified method is used instead:
 498  *
 499  * All hardware endpoints can be assigned a transfer resource and this setting
 500  * will stay persistent until either a core reset or hibernation. So whenever we
 501  * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
 502  * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
 503  * guaranteed that there are as many transfer resources as endpoints.
 504  *
 505  * This function is called for each endpoint when it is being enabled but is
 506  * triggered only when called for EP0-out, which always happens first, and which
 507  * should only happen in one of the above conditions.
 508  */
 509 static int dwc3_gadget_start_config(struct dwc3_ep *dep)
 510 {
 511         struct dwc3_gadget_ep_cmd_params params;
 512         struct dwc3             *dwc;
 513         u32                     cmd;
 514         int                     i;
 515         int                     ret;
 516
 517         if (dep->number)
 518                 return 0;
 519
 520         memset(&params, 0x00, sizeof(params));
 521         cmd = DWC3_DEPCMD_DEPSTARTCFG;
 522         dwc = dep->dwc;
 523
 524         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 525         if (ret)
 526                 return ret;
 527
 528         for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
 529                 struct dwc3_ep *dep = dwc->eps[i];
 530
 531                 if (!dep)
 532                         continue;
 533
 534                 ret = dwc3_gadget_set_xfer_resource(dep);
 535                 if (ret)
 536                         return ret;
 537         }
 538
 539         return 0;
 540 }
 541
 542 static void stream_timeout_function(struct timer_list *arg)
 543 {
 544         struct dwc3_request     *req = from_timer(req, arg, stream_timeout_timer);
 545         struct dwc3_ep          *dep = req->dep;
 546         struct dwc3             *dwc = dep->dwc;
 547         unsigned long           flags;
 548
 549         spin_lock_irqsave(&dwc->lock, flags);
 550         dwc3_stop_active_transfer(dep, true);
 551         __dwc3_gadget_kick_transfer(dep);
 552         spin_unlock_irqrestore(&dwc->lock, flags);
 553 }
 554
 555 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
 556 {
 557         const struct usb_ss_ep_comp_descriptor *comp_desc;
 558         const struct usb_endpoint_descriptor *desc;
 559         struct dwc3_gadget_ep_cmd_params params;
 560         struct dwc3 *dwc = dep->dwc;
 561
 562         comp_desc = dep->endpoint.comp_desc;
 563         desc = dep->endpoint.desc;
 564
 565         memset(&params, 0x00, sizeof(params));
 566
 567         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 568                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
 569
 570         /* Burst size is only needed in SuperSpeed mode */
 571         if (dwc->gadget.speed >= USB_SPEED_SUPER) {
 572                 u32 burst = dep->endpoint.maxburst;
 573                 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
 574         }
 575
 576         params.param0 |= action;
 577         if (action == DWC3_DEPCFG_ACTION_RESTORE)
 578                 params.param2 |= dep->saved_state;
 579
 580         if (usb_endpoint_xfer_control(desc))
 581                 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
 582
 583         if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
 584                 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
 585
 586         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 587                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 588                         | DWC3_DEPCFG_STREAM_EVENT_EN
 589                         | DWC3_DEPCFG_XFER_COMPLETE_EN;
 590                 dep->stream_capable = true;
 591         }
 592
 593         if (!usb_endpoint_xfer_control(desc))
 594                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 595
 596         /*
 597          * We are doing 1:1 mapping for endpoints, meaning
 598          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 599          * so on. We consider the direction bit as part of the physical
 600          * endpoint number. So USB endpoint 0x81 is 0x03.
 601          */
 602         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 603
 604         /*
 605          * We must use the lower 16 TX FIFOs even though
 606          * HW might have more
 607          */
 608         if (dep->direction)
 609                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 610
 611         if (desc->bInterval) {
 612                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
 613                 dep->interval = 1 << (desc->bInterval - 1);
 614         }
 615
 616         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
 617 }
 618
 619 /**
 620  * __dwc3_gadget_ep_enable - initializes a hw endpoint
 621  * @dep: endpoint to be initialized
 622  * @action: one of INIT, MODIFY or RESTORE
 623  *
 624  * Caller should take care of locking. Execute all necessary commands to
 625  * initialize a HW endpoint so it can be used by a gadget driver.
 626  */
 627 int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
 628 {
 629         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 630         struct dwc3             *dwc = dep->dwc;
 631
 632         u32                     reg;
 633         int                     ret;
 634
 635         if (!(dep->flags & DWC3_EP_ENABLED) || dwc->is_hibernated) {
 636                 ret = dwc3_gadget_start_config(dep);
 637                 if (ret)
 638                         return ret;
 639         }
 640
 641         ret = dwc3_gadget_set_ep_config(dep, action);
 642         if (ret)
 643                 return ret;
 644
 645         if (!(dep->flags & DWC3_EP_ENABLED) || dwc->is_hibernated) {
 646                 struct dwc3_trb *trb_st_hw;
 647                 struct dwc3_trb *trb_link;
 648
 649                 dep->type = usb_endpoint_type(desc);
 650                 dep->flags |= DWC3_EP_ENABLED;
 651                 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
 652
 653                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 654                 reg |= DWC3_DALEPENA_EP(dep->number);
 655                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 656
 657                 init_waitqueue_head(&dep->wait_end_transfer);
 658
 659                 if (usb_endpoint_xfer_control(desc))
 660                         goto out;
 661
 662                 if (!dwc->is_hibernated) {
 663                         /* Initialize the TRB ring */
 664                         dep->trb_dequeue = 0;
 665                         dep->trb_enqueue = 0;
 666                         memset(dep->trb_pool, 0,
 667                                sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
 668                 }
 669
 670                 /* Link TRB. The HWO bit is never reset */
 671                 trb_st_hw = &dep->trb_pool[0];
 672
 673                 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
 674                 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 675                 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 676                 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
 677                 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 678         }
 679
 680         /*
 681          * Issue StartTransfer here with no-op TRB so we can always rely on No
 682          * Response Update Transfer command.
 683          */
 684         if (((usb_endpoint_xfer_bulk(desc) && !dep->stream_capable) ||
 685              usb_endpoint_xfer_int(desc)) && !dwc->is_hibernated) {
 686                 struct dwc3_gadget_ep_cmd_params params;
 687                 struct dwc3_trb *trb;
 688                 dma_addr_t trb_dma;
 689                 u32 cmd;
 690
 691                 memset(&params, 0, sizeof(params));
 692                 trb = &dep->trb_pool[0];
 693                 trb_dma = dwc3_trb_dma_offset(dep, trb);
 694
 695                 params.param0 = upper_32_bits(trb_dma);
 696                 params.param1 = lower_32_bits(trb_dma);
 697
 698                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 699
 700                 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 701                 if (ret < 0)
 702                         return ret;
 703         }
 704
 705 out:
 706         trace_dwc3_gadget_ep_enable(dep);
 707
 708         return 0;
 709 }
 710
 711 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 712 {
 713         struct dwc3_request             *req;
 714
 715         dwc3_stop_active_transfer(dep, true);
 716
 717         /* - giveback all requests to gadget driver */
 718         while (!list_empty(&dep->started_list)) {
 719                 req = next_request(&dep->started_list);
 720
 721                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 722         }
 723
 724         while (!list_empty(&dep->pending_list)) {
 725                 req = next_request(&dep->pending_list);
 726
 727                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 728         }
 729 }
 730
 731 /**
 732  * __dwc3_gadget_ep_disable - disables a hw endpoint
 733  * @dep: the endpoint to disable
 734  *
 735  * This function undoes what __dwc3_gadget_ep_enable did and also removes
 736  * requests which are currently being processed by the hardware and those which
 737  * are not yet scheduled.
 738  *
 739  * Caller should take care of locking.
 740  */
 741 int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 742 {
 743         struct dwc3             *dwc = dep->dwc;
 744         u32                     reg;
 745
 746         trace_dwc3_gadget_ep_disable(dep);
 747
 748         dwc3_remove_requests(dwc, dep);
 749
 750         /* make sure HW endpoint isn't stalled */
 751         if (dep->flags & DWC3_EP_STALL)
 752                 __dwc3_gadget_ep_set_halt(dep, 0, false);
 753
 754         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 755         reg &= ~DWC3_DALEPENA_EP(dep->number);
 756         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 757
 758         dep->stream_capable = false;
 759         dep->type = 0;
 760         dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
 761
 762         /* Clear out the ep descriptors for non-ep0 */
 763         if (dep->number > 1) {
 764                 dep->endpoint.comp_desc = NULL;
 765                 dep->endpoint.desc = NULL;
 766         }
 767
 768         return 0;
 769 }
 770
 771 /* -------------------------------------------------------------------------- */
 772
 773 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 774                 const struct usb_endpoint_descriptor *desc)
 775 {
 776         return -EINVAL;
 777 }
 778
 779 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 780 {
 781         return -EINVAL;
 782 }
 783
 784 /* -------------------------------------------------------------------------- */
 785
 786 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 787                 const struct usb_endpoint_descriptor *desc)
 788 {
 789         struct dwc3_ep                  *dep;
 790         struct dwc3                     *dwc;
 791         unsigned long                   flags;
 792         int                             ret;
 793
 794         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 795                 pr_debug("dwc3: invalid parameters\n");
 796                 return -EINVAL;
 797         }
 798
 799         if (!desc->wMaxPacketSize) {
 800                 pr_debug("dwc3: missing wMaxPacketSize\n");
 801                 return -EINVAL;
 802         }
 803
 804         dep = to_dwc3_ep(ep);
 805         dwc = dep->dwc;
 806
 807         if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
 808                                         "%s is already enabled\n",
 809                                         dep->name))
 810                 return 0;
 811
 812         spin_lock_irqsave(&dwc->lock, flags);
 813         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
 814         spin_unlock_irqrestore(&dwc->lock, flags);
 815
 816         return ret;
 817 }
 818
 819 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 820 {
 821         struct dwc3_ep                  *dep;
 822         struct dwc3                     *dwc;
 823         unsigned long                   flags;
 824         int                             ret;
 825
 826         if (!ep) {
 827                 pr_debug("dwc3: invalid parameters\n");
 828                 return -EINVAL;
 829         }
 830
 831         dep = to_dwc3_ep(ep);
 832         dwc = dep->dwc;
 833
 834         if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
 835                                         "%s is already disabled\n",
 836                                         dep->name))
 837                 return 0;
 838
 839         spin_lock_irqsave(&dwc->lock, flags);
 840         ret = __dwc3_gadget_ep_disable(dep);
 841         spin_unlock_irqrestore(&dwc->lock, flags);
 842
 843         return ret;
 844 }
 845
 846 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 847                 gfp_t gfp_flags)
 848 {
 849         struct dwc3_request             *req;
 850         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 851
 852         req = kzalloc(sizeof(*req), gfp_flags);
 853         if (!req)
 854                 return NULL;
 855
 856         req->direction  = dep->direction;
 857         req->epnum      = dep->number;
 858         req->dep        = dep;
 859
 860         trace_dwc3_alloc_request(req);
 861
 862         return &req->request;
 863 }
 864
 865 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 866                 struct usb_request *request)
 867 {
 868         struct dwc3_request             *req = to_dwc3_request(request);
 869
 870         trace_dwc3_free_request(req);
 871         kfree(req);
 872 }
 873
 874 /**
 875  * dwc3_ep_prev_trb - returns the previous TRB in the ring
 876  * @dep: The endpoint with the TRB ring
 877  * @index: The index of the current TRB in the ring
 878  *
 879  * Returns the TRB prior to the one pointed to by the index. If the
 880  * index is 0, we will wrap backwards, skip the link TRB, and return
 881  * the one just before that.
 882  */
 883 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
 884 {
 885         u8 tmp = index;
 886
 887         if (!tmp)
 888                 tmp = DWC3_TRB_NUM - 1;
 889
 890         return &dep->trb_pool[tmp - 1];
 891 }
 892
 893 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
 894 {
 895         struct dwc3_trb         *tmp;
 896         u8                      trbs_left;
 897
 898         /*
 899          * If enqueue & dequeue are equal than it is either full or empty.
 900          *
 901          * One way to know for sure is if the TRB right before us has HWO bit
 902          * set or not. If it has, then we're definitely full and can't fit any
 903          * more transfers in our ring.
 904          */
 905         if (dep->trb_enqueue == dep->trb_dequeue) {
 906                 tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
 907                 if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
 908                         return 0;
 909
 910                 return DWC3_TRB_NUM - 1;
 911         }
 912
 913         trbs_left = dep->trb_dequeue - dep->trb_enqueue;
 914         trbs_left &= (DWC3_TRB_NUM - 1);
 915
 916         if (dep->trb_dequeue < dep->trb_enqueue)
 917                 trbs_left--;
 918
 919         return trbs_left;
 920 }
 921
 922 static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
 923                 dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
 924                 unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt)
 925 {
 926         struct dwc3             *dwc = dep->dwc;
 927         struct usb_gadget       *gadget = &dwc->gadget;
 928         enum usb_device_speed   speed = gadget->speed;
 929
 930         trb->size = DWC3_TRB_SIZE_LENGTH(length);
 931         trb->bpl = lower_32_bits(dma);
 932         trb->bph = upper_32_bits(dma);
 933
 934         switch (usb_endpoint_type(dep->endpoint.desc)) {
 935         case USB_ENDPOINT_XFER_CONTROL:
 936                 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
 937                 break;
 938
 939         case USB_ENDPOINT_XFER_ISOC:
 940                 if (!node) {
 941                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 942
 943                         /*
 944                          * USB Specification 2.0 Section 5.9.2 states that: "If
 945                          * there is only a single transaction in the microframe,
 946                          * only a DATA0 data packet PID is used.  If there are
 947                          * two transactions per microframe, DATA1 is used for
 948                          * the first transaction data packet and DATA0 is used
 949                          * for the second transaction data packet.  If there are
 950                          * three transactions per microframe, DATA2 is used for
 951                          * the first transaction data packet, DATA1 is used for
 952                          * the second, and DATA0 is used for the third."
 953                          *
 954                          * IOW, we should satisfy the following cases:
 955                          *
 956                          * 1) length <= maxpacket
 957                          *      - DATA0
 958                          *
 959                          * 2) maxpacket < length <= (2 * maxpacket)
 960                          *      - DATA1, DATA0
 961                          *
 962                          * 3) (2 * maxpacket) < length <= (3 * maxpacket)
 963                          *      - DATA2, DATA1, DATA0
 964                          */
 965                         if (speed == USB_SPEED_HIGH) {
 966                                 struct usb_ep *ep = &dep->endpoint;
 967                                 unsigned int mult = 2;
 968                                 unsigned int maxp = usb_endpoint_maxp(ep->desc);
 969
 970                                 if (length <= (2 * maxp))
 971                                         mult--;
 972
 973                                 if (length <= maxp)
 974                                         mult--;
 975
 976                                 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
 977                         }
 978                 } else {
 979                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
 980                 }
 981
 982                 /* always enable Interrupt on Missed ISOC */
 983                 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
 984                 break;
 985
 986         case USB_ENDPOINT_XFER_BULK:
 987         case USB_ENDPOINT_XFER_INT:
 988                 trb->ctrl = DWC3_TRBCTL_NORMAL;
 989                 break;
 990         default:
 991                 /*
 992                  * This is only possible with faulty memory because we
 993                  * checked it already :)
 994                  */
 995                 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
 996                                 usb_endpoint_type(dep->endpoint.desc));
 997         }
 998
 999         /* always enable Continue on Short Packet */
1000         if (usb_endpoint_dir_out(dep->endpoint.desc)) {
1001                 trb->ctrl |= DWC3_TRB_CTRL_CSP;
1002
1003                 if (short_not_ok)
1004                         trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1005         }
1006
1007         if ((!no_interrupt && !chain) ||
1008             (dwc3_calc_trbs_left(dep) == 1))
1009                 trb->ctrl |= DWC3_TRB_CTRL_IOC;
1010
1011         if (chain)
1012                 trb->ctrl |= DWC3_TRB_CTRL_CHN;
1013         /*
1014          * To start transfer on another stream number endpoint need to relase
1015          * previously acquired transfer resource for doing that there is two
1016          * ways 1. end transfer 2. set lst bit of control trb
1017          *
1018          * by using lst bit in ctrl trb we will be able to save the time of
1019          * ending transfer hence improved performance
1020          */
1021         else if (dep->stream_capable)
1022                 trb->ctrl |= DWC3_TRB_CTRL_LST;
1023
1024         if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1025                 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1026
1027         trb->ctrl |= DWC3_TRB_CTRL_HWO;
1028
1029         dwc3_ep_inc_enq(dep);
1030
1031         trace_dwc3_prepare_trb(dep, trb);
1032 }
1033
1034 /**
1035  * dwc3_prepare_one_trb - setup one TRB from one request
1036  * @dep: endpoint for which this request is prepared
1037  * @req: dwc3_request pointer
1038  * @chain: should this TRB be chained to the next?
1039  * @node: only for isochronous endpoints. First TRB needs different type.
1040  */
1041 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1042                 struct dwc3_request *req, unsigned chain, unsigned node)
1043 {
1044         struct dwc3_trb         *trb;
1045         unsigned int            length;
1046         dma_addr_t              dma;
1047         unsigned                stream_id = req->request.stream_id;
1048         unsigned                short_not_ok = req->request.short_not_ok;
1049         unsigned                no_interrupt = req->request.no_interrupt;
1050
1051         if (req->request.num_sgs > 0) {
1052                 length = sg_dma_len(req->start_sg);
1053                 dma = sg_dma_address(req->start_sg);
1054         } else {
1055                 length = req->request.length;
1056                 dma = req->request.dma;
1057         }
1058
1059         trb = &dep->trb_pool[dep->trb_enqueue];
1060
1061         if (!req->trb) {
1062                 dwc3_gadget_move_started_request(req);
1063                 req->trb = trb;
1064                 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1065         }
1066
1067         __dwc3_prepare_one_trb(dep, trb, dma, length, chain, node,
1068                         stream_id, short_not_ok, no_interrupt);
1069 }
1070
1071 static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
1072                 struct dwc3_request *req)
1073 {
1074         struct scatterlist *sg = req->start_sg;
1075         struct scatterlist *s;
1076         int             i;
1077
1078         unsigned int remaining = req->request.num_mapped_sgs
1079                 - req->num_queued_sgs;
1080
1081         for_each_sg(sg, s, remaining, i) {
1082                 unsigned int length = req->request.length;
1083                 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1084                 unsigned int rem = length % maxp;
1085                 unsigned chain = true;
1086
1087                 if (sg_is_last(s))
1088                         chain = false;
1089
1090                 if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) {
1091                         struct dwc3     *dwc = dep->dwc;
1092                         struct dwc3_trb *trb;
1093
1094                         req->unaligned = true;
1095
1096                         /* prepare normal TRB */
1097                         dwc3_prepare_one_trb(dep, req, true, i);
1098
1099                         /* Now prepare one extra TRB to align transfer size */
1100                         trb = &dep->trb_pool[dep->trb_enqueue];
1101                         __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr,
1102                                         maxp - rem, false, 0,
1103                                         req->request.stream_id,
1104                                         req->request.short_not_ok,
1105                                         req->request.no_interrupt);
1106                 } else {
1107                         dwc3_prepare_one_trb(dep, req, chain, i);
1108                 }
1109
1110                 /*
1111                  * There can be a situation where all sgs in sglist are not
1112                  * queued because of insufficient trb number. To handle this
1113                  * case, update start_sg to next sg to be queued, so that
1114                  * we have free trbs we can continue queuing from where we
1115                  * previously stopped
1116                  */
1117                 if (chain)
1118                         req->start_sg = sg_next(s);
1119
1120                 req->num_queued_sgs++;
1121
1122                 if (!dwc3_calc_trbs_left(dep))
1123                         break;
1124         }
1125 }
1126
1127 static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
1128                 struct dwc3_request *req)
1129 {
1130         unsigned int length = req->request.length;
1131         unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1132         unsigned int rem = length % maxp;
1133
1134         if (rem && usb_endpoint_dir_out(dep->endpoint.desc)) {
1135                 struct dwc3     *dwc = dep->dwc;
1136                 struct dwc3_trb *trb;
1137
1138                 req->unaligned = true;
1139
1140                 /* prepare normal TRB */
1141                 dwc3_prepare_one_trb(dep, req, true, 0);
1142
1143                 /* Now prepare one extra TRB to align transfer size */
1144                 trb = &dep->trb_pool[dep->trb_enqueue];
1145                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
1146                                 false, 0, req->request.stream_id,
1147                                 req->request.short_not_ok,
1148                                 req->request.no_interrupt);
1149         } else if (req->request.zero && req->request.length &&
1150                    (IS_ALIGNED(req->request.length, maxp))) {
1151                 struct dwc3     *dwc = dep->dwc;
1152                 struct dwc3_trb *trb;
1153
1154                 req->zero = true;
1155
1156                 /* prepare normal TRB */
1157                 dwc3_prepare_one_trb(dep, req, true, 0);
1158
1159                 /* Now prepare one extra TRB to handle ZLP */
1160                 trb = &dep->trb_pool[dep->trb_enqueue];
1161                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1162                                 false, 0, req->request.stream_id,
1163                                 req->request.short_not_ok,
1164                                 req->request.no_interrupt);
1165         } else {
1166                 dwc3_prepare_one_trb(dep, req, false, 0);
1167         }
1168 }
1169
1170 /*
1171  * dwc3_prepare_trbs - setup TRBs from requests
1172  * @dep: endpoint for which requests are being prepared
1173  *
1174  * The function goes through the requests list and sets up TRBs for the
1175  * transfers. The function returns once there are no more TRBs available or
1176  * it runs out of requests.
1177  */
1178 static void dwc3_prepare_trbs(struct dwc3_ep *dep)
1179 {
1180         struct dwc3_request     *req, *n;
1181
1182         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1183
1184         /*
1185          * We can get in a situation where there's a request in the started list
1186          * but there weren't enough TRBs to fully kick it in the first time
1187          * around, so it has been waiting for more TRBs to be freed up.
1188          *
1189          * In that case, we should check if we have a request with pending_sgs
1190          * in the started list and prepare TRBs for that request first,
1191          * otherwise we will prepare TRBs completely out of order and that will
1192          * break things.
1193          */
1194         list_for_each_entry(req, &dep->started_list, list) {
1195                 if (req->num_pending_sgs > 0)
1196                         dwc3_prepare_one_trb_sg(dep, req);
1197
1198                 if (!dwc3_calc_trbs_left(dep))
1199                         return;
1200         }
1201
1202         list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1203                 struct dwc3     *dwc = dep->dwc;
1204                 int             ret;
1205
1206                 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1207                                                     dep->direction);
1208                 if (ret)
1209                         return;
1210
1211                 req->sg                 = req->request.sg;
1212                 req->start_sg           = req->sg;
1213                 req->num_queued_sgs     = 0;
1214                 req->num_pending_sgs    = req->request.num_mapped_sgs;
1215
1216                 if (req->num_pending_sgs > 0)
1217                         dwc3_prepare_one_trb_sg(dep, req);
1218                 else
1219                         dwc3_prepare_one_trb_linear(dep, req);
1220
1221                 if (!dwc3_calc_trbs_left(dep))
1222                         return;
1223         }
1224 }
1225
1226 int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1227 {
1228         struct dwc3_gadget_ep_cmd_params params;
1229         struct dwc3_request             *req;
1230         int                             starting;
1231         int                             ret;
1232         u32                             cmd;
1233
1234         if (!dwc3_calc_trbs_left(dep))
1235                 return 0;
1236
1237         starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1238
1239         dwc3_prepare_trbs(dep);
1240         req = next_request(&dep->started_list);
1241         if (!req) {
1242                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1243                 return 0;
1244         }
1245
1246         memset(&params, 0, sizeof(params));
1247
1248         if (starting) {
1249                 params.param0 = upper_32_bits(req->trb_dma);
1250                 params.param1 = lower_32_bits(req->trb_dma);
1251                 cmd = DWC3_DEPCMD_STARTTRANSFER;
1252
1253                 if (dep->stream_capable)
1254                         cmd = cmd | DWC3_DEPCMD_PARAM(req->request.stream_id);
1255
1256                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1257                         cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1258
1259         } else {
1260                 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1261                         DWC3_DEPCMD_PARAM(dep->resource_index);
1262         }
1263
1264         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1265         if (ret < 0) {
1266                 /*
1267                  * FIXME we need to iterate over the list of requests
1268                  * here and stop, unmap, free and del each of the linked
1269                  * requests instead of what we do now.
1270                  */
1271                 if (req->trb)
1272                         memset(req->trb, 0, sizeof(struct dwc3_trb));
1273                 dwc3_gadget_del_and_unmap_request(dep, req, ret);
1274                 return ret;
1275         }
1276
1277         if (starting && dep->stream_capable) {
1278                 req->stream_timeout_timer.expires = jiffies +
1279                                 msecs_to_jiffies(STREAM_TIMEOUT_MS);
1280                 mod_timer(&req->stream_timeout_timer,
1281                           req->stream_timeout_timer.expires);
1282         }
1283
1284         return 0;
1285 }
1286
1287 static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1288 {
1289         u32                     reg;
1290
1291         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1292         return DWC3_DSTS_SOFFN(reg);
1293 }
1294
1295 static void __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1296 {
1297         if (list_empty(&dep->pending_list)) {
1298                 dev_info(dep->dwc->dev, "%s: ran out of requests\n",
1299                                 dep->name);
1300                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1301                 return;
1302         }
1303
1304         dep->frame_number = DWC3_ALIGN_FRAME(dep);
1305         __dwc3_gadget_kick_transfer(dep);
1306         dep->flags &= ~DWC3_EP_PENDING_REQUEST;
1307 }
1308
1309 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc);
1310 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1311 {
1312         struct dwc3             *dwc = dep->dwc;
1313
1314         if (!dep->endpoint.desc) {
1315                 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
1316                                 dep->name);
1317                 return -ESHUTDOWN;
1318         }
1319
1320         if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1321                                 &req->request, req->dep->name))
1322                 return -EINVAL;
1323
1324         pm_runtime_get(dwc->dev);
1325
1326         req->request.actual     = 0;
1327         req->request.status     = -EINPROGRESS;
1328
1329         if (dep->stream_capable)
1330                 timer_setup(&req->stream_timeout_timer,
1331                             stream_timeout_function, 0);
1332
1333         trace_dwc3_ep_queue(req);
1334
1335         list_add_tail(&req->list, &dep->pending_list);
1336
1337         /* If core is hibernated, need to wakeup (remote wakeup) */
1338         if (dwc->is_hibernated) {
1339                 dwc->force_hiber_wake = true;
1340                 gadget_wakeup_interrupt(dwc);
1341                 dwc->force_hiber_wake = false;
1342         }
1343
1344         /*
1345          * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1346          * wait for a XferNotReady event so we will know what's the current
1347          * (micro-)frame number.
1348          *
1349          * Without this trick, we are very, very likely gonna get Bus Expiry
1350          * errors which will force us issue EndTransfer command.
1351          */
1352         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1353                 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1354                         if (dep->flags & DWC3_EP_TRANSFER_STARTED) {
1355                                 dwc3_stop_active_transfer(dep, true);
1356                                 dep->flags = DWC3_EP_ENABLED;
1357                         } else {
1358                                 u32 cur_uf;
1359
1360                                 cur_uf = __dwc3_gadget_get_frame(dwc);
1361                                 __dwc3_gadget_start_isoc(dep);
1362                                 dep->flags &= ~DWC3_EP_PENDING_REQUEST;
1363                         }
1364                         return 0;
1365                 }
1366
1367                 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1368                         if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1369                                 __dwc3_gadget_start_isoc(dep);
1370                                 return 0;
1371                         }
1372                 }
1373         }
1374
1375         return __dwc3_gadget_kick_transfer(dep);
1376 }
1377
1378 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1379         gfp_t gfp_flags)
1380 {
1381         struct dwc3_request             *req = to_dwc3_request(request);
1382         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1383         struct dwc3                     *dwc = dep->dwc;
1384
1385         unsigned long                   flags;
1386
1387         int                             ret;
1388
1389         spin_lock_irqsave(&dwc->lock, flags);
1390         ret = __dwc3_gadget_ep_queue(dep, req);
1391         spin_unlock_irqrestore(&dwc->lock, flags);
1392
1393         return ret;
1394 }
1395
1396 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1397                 struct usb_request *request)
1398 {
1399         struct dwc3_request             *req = to_dwc3_request(request);
1400         struct dwc3_request             *r = NULL;
1401
1402         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1403         struct dwc3                     *dwc = dep->dwc;
1404
1405         unsigned long                   flags;
1406         int                             ret = 0;
1407
1408         trace_dwc3_ep_dequeue(req);
1409
1410         spin_lock_irqsave(&dwc->lock, flags);
1411
1412         if (dep->stream_capable && timer_pending(&req->stream_timeout_timer))
1413                 del_timer(&req->stream_timeout_timer);
1414
1415         list_for_each_entry(r, &dep->pending_list, list) {
1416                 if (r == req)
1417                         break;
1418         }
1419
1420         if (r != req) {
1421                 list_for_each_entry(r, &dep->started_list, list) {
1422                         if (r == req)
1423                                 break;
1424                 }
1425                 if (r == req) {
1426                         /* wait until it is processed */
1427                         dwc3_stop_active_transfer(dep, true);
1428
1429                         /*
1430                          * If request was already started, this means we had to
1431                          * stop the transfer. With that we also need to ignore
1432                          * all TRBs used by the request, however TRBs can only
1433                          * be modified after completion of END_TRANSFER
1434                          * command. So what we do here is that we wait for
1435                          * END_TRANSFER completion and only after that, we jump
1436                          * over TRBs by clearing HWO and incrementing dequeue
1437                          * pointer.
1438                          *
1439                          * Note that we have 2 possible types of transfers here:
1440                          *
1441                          * i) Linear buffer request
1442                          * ii) SG-list based request
1443                          *
1444                          * SG-list based requests will have r->num_pending_sgs
1445                          * set to a valid number (> 0). Linear requests,
1446                          * normally use a single TRB.
1447                          *
1448                          * For each of these two cases, if r->unaligned flag is
1449                          * set, one extra TRB has been used to align transfer
1450                          * size to wMaxPacketSize.
1451                          *
1452                          * All of these cases need to be taken into
1453                          * consideration so we don't mess up our TRB ring
1454                          * pointers.
1455                          */
1456                         wait_event_lock_irq(dep->wait_end_transfer,
1457                                         !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
1458                                         dwc->lock);
1459
1460                         if (!r->trb)
1461                                 goto out0;
1462
1463                         if (r->num_pending_sgs) {
1464                                 struct dwc3_trb *trb;
1465                                 int i = 0;
1466
1467                                 for (i = 0; i < r->num_pending_sgs; i++) {
1468                                         trb = r->trb + i;
1469                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1470                                         dwc3_ep_inc_deq(dep);
1471                                 }
1472
1473                                 if (r->unaligned || r->zero) {
1474                                         trb = r->trb + r->num_pending_sgs + 1;
1475                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1476                                         dwc3_ep_inc_deq(dep);
1477                                 }
1478                         } else {
1479                                 struct dwc3_trb *trb = r->trb;
1480
1481                                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1482                                 dwc3_ep_inc_deq(dep);
1483
1484                                 if (r->unaligned || r->zero) {
1485                                         trb = r->trb + 1;
1486                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1487                                         dwc3_ep_inc_deq(dep);
1488                                 }
1489                         }
1490                         goto out1;
1491                 }
1492                 dev_err(dwc->dev, "request %pK was not queued to %s\n",
1493                                 request, ep->name);
1494                 ret = -EINVAL;
1495                 goto out0;
1496         }
1497
1498 out1:
1499         /* giveback the request */
1500
1501         dwc3_gadget_giveback(dep, req, -ECONNRESET);
1502
1503 out0:
1504         spin_unlock_irqrestore(&dwc->lock, flags);
1505
1506         return ret;
1507 }
1508
1509 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1510 {
1511         struct dwc3_gadget_ep_cmd_params        params;
1512         struct dwc3                             *dwc = dep->dwc;
1513         int                                     ret;
1514
1515         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1516                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1517                 return -EINVAL;
1518         }
1519
1520         memset(&params, 0x00, sizeof(params));
1521
1522         if (value) {
1523                 struct dwc3_trb *trb;
1524
1525                 unsigned transfer_in_flight;
1526                 unsigned started;
1527
1528                 if (dep->flags & DWC3_EP_STALL)
1529                         return 0;
1530
1531                 if (dep->number > 1)
1532                         trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1533                 else
1534                         trb = &dwc->ep0_trb[dep->trb_enqueue];
1535
1536                 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
1537                 started = !list_empty(&dep->started_list);
1538
1539                 if (!protocol && ((dep->direction && transfer_in_flight) ||
1540                                 (!dep->direction && started))) {
1541                         return -EAGAIN;
1542                 }
1543
1544                 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1545                                 &params);
1546                 if (ret)
1547                         dev_err(dwc->dev, "failed to set STALL on %s\n",
1548                                         dep->name);
1549                 else
1550                         dep->flags |= DWC3_EP_STALL;
1551         } else {
1552                 if (!(dep->flags & DWC3_EP_STALL))
1553                         return 0;
1554
1555                 ret = dwc3_send_clear_stall_ep_cmd(dep);
1556                 if (ret)
1557                         dev_err(dwc->dev, "failed to clear STALL on %s\n",
1558                                         dep->name);
1559                 else
1560                         dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1561         }
1562
1563         return ret;
1564 }
1565
1566 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1567 {
1568         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1569         struct dwc3                     *dwc = dep->dwc;
1570
1571         unsigned long                   flags;
1572
1573         int                             ret;
1574
1575         spin_lock_irqsave(&dwc->lock, flags);
1576         ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1577         spin_unlock_irqrestore(&dwc->lock, flags);
1578
1579         return ret;
1580 }
1581
1582 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1583 {
1584         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1585         struct dwc3                     *dwc = dep->dwc;
1586         unsigned long                   flags;
1587         int                             ret;
1588
1589         spin_lock_irqsave(&dwc->lock, flags);
1590         dep->flags |= DWC3_EP_WEDGE;
1591
1592         if (dep->number == 0 || dep->number == 1)
1593                 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1594         else
1595                 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1596         spin_unlock_irqrestore(&dwc->lock, flags);
1597
1598         return ret;
1599 }
1600
1601 /* -------------------------------------------------------------------------- */
1602
1603 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1604         .bLength        = USB_DT_ENDPOINT_SIZE,
1605         .bDescriptorType = USB_DT_ENDPOINT,
1606         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
1607 };
1608
1609 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1610         .enable         = dwc3_gadget_ep0_enable,
1611         .disable        = dwc3_gadget_ep0_disable,
1612         .alloc_request  = dwc3_gadget_ep_alloc_request,
1613         .free_request   = dwc3_gadget_ep_free_request,
1614         .queue          = dwc3_gadget_ep0_queue,
1615         .dequeue        = dwc3_gadget_ep_dequeue,
1616         .set_halt       = dwc3_gadget_ep0_set_halt,
1617         .set_wedge      = dwc3_gadget_ep_set_wedge,
1618 };
1619
1620 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1621         .enable         = dwc3_gadget_ep_enable,
1622         .disable        = dwc3_gadget_ep_disable,
1623         .alloc_request  = dwc3_gadget_ep_alloc_request,
1624         .free_request   = dwc3_gadget_ep_free_request,
1625         .queue          = dwc3_gadget_ep_queue,
1626         .dequeue        = dwc3_gadget_ep_dequeue,
1627         .set_halt       = dwc3_gadget_ep_set_halt,
1628         .set_wedge      = dwc3_gadget_ep_set_wedge,
1629 };
1630
1631 /* -------------------------------------------------------------------------- */
1632
1633 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1634 {
1635         struct dwc3             *dwc = gadget_to_dwc(g);
1636
1637         return __dwc3_gadget_get_frame(dwc);
1638 }
1639
1640 static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
1641 {
1642         int                     retries;
1643
1644         int                     ret;
1645         u32                     reg;
1646
1647         u8                      link_state;
1648         u8                      speed;
1649
1650         /*
1651          * According to the Databook Remote wakeup request should
1652          * be issued only when the device is in early suspend state.
1653          *
1654          * We can check that via USB Link State bits in DSTS register.
1655          */
1656         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1657
1658         speed = reg & DWC3_DSTS_CONNECTSPD;
1659         if ((speed == DWC3_DSTS_SUPERSPEED) ||
1660             (speed == DWC3_DSTS_SUPERSPEED_PLUS))
1661                 return 0;
1662
1663         link_state = DWC3_DSTS_USBLNKST(reg);
1664
1665         switch (link_state) {
1666         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1667         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1668                 break;
1669         default:
1670                 return -EINVAL;
1671         }
1672
1673         ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1674         if (ret < 0) {
1675                 dev_err(dwc->dev, "failed to put link in Recovery\n");
1676                 return ret;
1677         }
1678
1679         /* Recent versions do this automatically */
1680         if (dwc->revision < DWC3_REVISION_194A) {
1681                 /* write zeroes to Link Change Request */
1682                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1683                 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1684                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1685         }
1686
1687         /* poll until Link State changes to ON */
1688         retries = 20000;
1689
1690         while (retries--) {
1691                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1692
1693                 /* in HS, means ON */
1694                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1695                         break;
1696         }
1697
1698         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1699                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1700                 return -EINVAL;
1701         }
1702
1703         return 0;
1704 }
1705
1706 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1707 {
1708         struct dwc3             *dwc = gadget_to_dwc(g);
1709         unsigned long           flags;
1710         int                     ret;
1711
1712         spin_lock_irqsave(&dwc->lock, flags);
1713         ret = __dwc3_gadget_wakeup(dwc);
1714         spin_unlock_irqrestore(&dwc->lock, flags);
1715
1716         return ret;
1717 }
1718
1719 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1720                 int is_selfpowered)
1721 {
1722         struct dwc3             *dwc = gadget_to_dwc(g);
1723         unsigned long           flags;
1724
1725         spin_lock_irqsave(&dwc->lock, flags);
1726         g->is_selfpowered = !!is_selfpowered;
1727         spin_unlock_irqrestore(&dwc->lock, flags);
1728
1729         return 0;
1730 }
1731
1732 int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1733 {
1734         u32                     reg;
1735         u32                     timeout = 500;
1736
1737         if (pm_runtime_suspended(dwc->dev))
1738                 return 0;
1739
1740         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1741         if (is_on) {
1742                 if (dwc->revision <= DWC3_REVISION_187A) {
1743                         reg &= ~DWC3_DCTL_TRGTULST_MASK;
1744                         reg |= DWC3_DCTL_TRGTULST_RX_DET;
1745                 }
1746
1747                 if (dwc->revision >= DWC3_REVISION_194A)
1748                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1749                 reg |= DWC3_DCTL_RUN_STOP;
1750
1751                 if (dwc->has_hibernation)
1752                         reg |= DWC3_DCTL_KEEP_CONNECT;
1753
1754                 dwc->pullups_connected = true;
1755         } else {
1756                 reg &= ~DWC3_DCTL_RUN_STOP;
1757
1758                 if (dwc->has_hibernation && !suspend)
1759                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1760
1761                 dwc->pullups_connected = false;
1762         }
1763
1764         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1765
1766         do {
1767                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1768                 reg &= DWC3_DSTS_DEVCTRLHLT;
1769         } while (--timeout && !(!is_on ^ !reg));
1770
1771         if (!timeout)
1772                 return -ETIMEDOUT;
1773
1774         return 0;
1775 }
1776
1777 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1778 {
1779         struct dwc3             *dwc = gadget_to_dwc(g);
1780         unsigned long           flags;
1781         int                     ret;
1782
1783         is_on = !!is_on;
1784
1785         /*
1786          * Per databook, when we want to stop the gadget, if a control transfer
1787          * is still in process, complete it and get the core into setup phase.
1788          */
1789         if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
1790                 reinit_completion(&dwc->ep0_in_setup);
1791
1792                 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
1793                                 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
1794                 if (ret == 0) {
1795                         dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
1796                         return -ETIMEDOUT;
1797                 }
1798         }
1799
1800         spin_lock_irqsave(&dwc->lock, flags);
1801         ret = dwc3_gadget_run_stop(dwc, is_on, false);
1802         spin_unlock_irqrestore(&dwc->lock, flags);
1803
1804         return ret;
1805 }
1806
1807 void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1808 {
1809         u32                     reg;
1810
1811         /* Enable all but Start and End of Frame IRQs */
1812         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1813                         DWC3_DEVTEN_EVNTOVERFLOWEN |
1814                         DWC3_DEVTEN_CMDCMPLTEN |
1815                         DWC3_DEVTEN_ERRTICERREN |
1816                         DWC3_DEVTEN_WKUPEVTEN |
1817                         DWC3_DEVTEN_CONNECTDONEEN |
1818                         DWC3_DEVTEN_USBRSTEN |
1819                         DWC3_DEVTEN_DISCONNEVTEN);
1820
1821         /* Enable hibernation IRQ */
1822         if (dwc->has_hibernation)
1823                 reg |= DWC3_DEVTEN_HIBERNATIONREQEVTEN;
1824
1825         if (dwc->revision < DWC3_REVISION_250A)
1826                 reg |= DWC3_DEVTEN_ULSTCNGEN;
1827
1828         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1829 }
1830
1831 void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1832 {
1833         /* mask all interrupts */
1834         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1835 }
1836
1837 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1838 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1839
1840 /**
1841  * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
1842  * @dwc: pointer to our context structure
1843  *
1844  * The following looks like complex but it's actually very simple. In order to
1845  * calculate the number of packets we can burst at once on OUT transfers, we're
1846  * gonna use RxFIFO size.
1847  *
1848  * To calculate RxFIFO size we need two numbers:
1849  * MDWIDTH = size, in bits, of the internal memory bus
1850  * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
1851  *
1852  * Given these two numbers, the formula is simple:
1853  *
1854  * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
1855  *
1856  * 24 bytes is for 3x SETUP packets
1857  * 16 bytes is a clock domain crossing tolerance
1858  *
1859  * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
1860  */
1861 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
1862 {
1863         u32 ram2_depth;
1864         u32 mdwidth;
1865         u32 nump;
1866         u32 reg;
1867
1868         ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
1869         mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
1870
1871         nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
1872         nump = min_t(u32, nump, 16);
1873
1874         /* update NumP */
1875         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1876         reg &= ~DWC3_DCFG_NUMP_MASK;
1877         reg |= nump << DWC3_DCFG_NUMP_SHIFT;
1878         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1879 }
1880
1881 static int __dwc3_gadget_start(struct dwc3 *dwc)
1882 {
1883         struct dwc3_ep          *dep;
1884         int                     ret = 0;
1885         u32                     reg;
1886
1887         /*
1888          * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
1889          * the core supports IMOD, disable it.
1890          */
1891         if (dwc->imod_interval) {
1892                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
1893                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
1894         } else if (dwc3_has_imod(dwc)) {
1895                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
1896         }
1897
1898         /*
1899          * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
1900          * field instead of letting dwc3 itself calculate that automatically.
1901          *
1902          * This way, we maximize the chances that we'll be able to get several
1903          * bursts of data without going through any sort of endpoint throttling.
1904          */
1905         reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
1906         if (dwc3_is_usb31(dwc))
1907                 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
1908         else
1909                 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
1910
1911         dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
1912
1913         dwc3_gadget_setup_nump(dwc);
1914
1915         /* For OTG mode, check if the core is currently in Host mode.
1916          * This is not an error condition as there are times when the core is
1917          * working as host and kernel is told to initiate bind operation with
1918          * gadget class driver module.
1919          * The below remaining operations are handled in OTG driver whenever
1920          * required.
1921          */
1922         if (dwc3_readl(dwc->regs, DWC3_GSTS) & DWC3_GSTS_CUR_MODE)
1923                 return 0;
1924
1925         /* Start with SuperSpeed Default */
1926         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1927
1928         dep = dwc->eps[0];
1929         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1930         if (ret) {
1931                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1932                 goto err0;
1933         }
1934
1935         dep = dwc->eps[1];
1936         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1937         if (ret) {
1938                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1939                 goto err1;
1940         }
1941
1942         /* begin to receive SETUP packets */
1943         dwc->ep0state = EP0_SETUP_PHASE;
1944         dwc3_ep0_out_start(dwc);
1945
1946         dwc3_gadget_enable_irq(dwc);
1947
1948         return 0;
1949
1950 err1:
1951         __dwc3_gadget_ep_disable(dwc->eps[0]);
1952
1953 err0:
1954         return ret;
1955 }
1956
1957 static irqreturn_t wakeup_interrupt(int irq, void *_dwc);
1958 static int dwc3_gadget_start(struct usb_gadget *g,
1959                 struct usb_gadget_driver *driver)
1960 {
1961         struct dwc3             *dwc = gadget_to_dwc(g);
1962         unsigned long           flags;
1963         int                     ret = 0;
1964         int                     irq;
1965
1966         irq = dwc->irq_gadget;
1967         ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1968                         IRQF_SHARED, "dwc3", dwc->ev_buf);
1969         if (ret) {
1970                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1971                                 irq, ret);
1972                 goto err0;
1973         }
1974
1975         /* look for wakeup interrupt if hibernation is supported */
1976         if (dwc->has_hibernation) {
1977                 irq = dwc->irq_wakeup;
1978                 ret = devm_request_irq(dwc->dev, irq, wakeup_interrupt,
1979                                        IRQF_SHARED, "usb-wakeup", dwc);
1980                 if (ret) {
1981                         dev_err(dwc->dev, "failed to request wakeup irq #%d --> %d\n",
1982                                 irq, ret);
1983                         goto err0;
1984                 }
1985         }
1986
1987         spin_lock_irqsave(&dwc->lock, flags);
1988         if (dwc->gadget_driver) {
1989                 dev_err(dwc->dev, "%s is already bound to %s\n",
1990                                 dwc->gadget.name,
1991                                 dwc->gadget_driver->driver.name);
1992                 ret = -EBUSY;
1993                 goto err1;
1994         }
1995
1996         dwc->gadget_driver      = driver;
1997
1998         if (pm_runtime_active(dwc->dev))
1999                 __dwc3_gadget_start(dwc);
2000
2001         spin_unlock_irqrestore(&dwc->lock, flags);
2002
2003         return 0;
2004
2005 err1:
2006         spin_unlock_irqrestore(&dwc->lock, flags);
2007         if (dwc->irq_gadget)
2008                 free_irq(dwc->irq_gadget, dwc->ev_buf);
2009         if (dwc->irq_wakeup)
2010                 free_irq(dwc->irq_wakeup, dwc);
2011
2012 err0:
2013         return ret;
2014 }
2015
2016 static void __dwc3_gadget_stop(struct dwc3 *dwc)
2017 {
2018         dwc3_gadget_disable_irq(dwc);
2019         __dwc3_gadget_ep_disable(dwc->eps[0]);
2020         __dwc3_gadget_ep_disable(dwc->eps[1]);
2021 }
2022
2023 static int dwc3_gadget_stop(struct usb_gadget *g)
2024 {
2025         struct dwc3             *dwc = gadget_to_dwc(g);
2026         unsigned long           flags;
2027         int                     epnum;
2028         u32                     tmo_eps = 0;
2029
2030         spin_lock_irqsave(&dwc->lock, flags);
2031
2032         if (pm_runtime_suspended(dwc->dev))
2033                 goto out;
2034
2035         __dwc3_gadget_stop(dwc);
2036
2037         for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2038                 struct dwc3_ep  *dep = dwc->eps[epnum];
2039                 int ret;
2040
2041                 if (!dep)
2042                         continue;
2043
2044                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
2045                         continue;
2046
2047                 ret = wait_event_interruptible_lock_irq_timeout(dep->wait_end_transfer,
2048                             !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
2049                             dwc->lock, msecs_to_jiffies(5));
2050
2051                 if (ret <= 0) {
2052                         /* Timed out or interrupted! There's nothing much
2053                          * we can do so we just log here and print which
2054                          * endpoints timed out at the end.
2055                          */
2056                         tmo_eps |= 1 << epnum;
2057                         dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
2058                 }
2059         }
2060
2061         if (tmo_eps) {
2062                 dev_err(dwc->dev,
2063                         "end transfer timed out on endpoints 0x%x [bitmap]\n",
2064                         tmo_eps);
2065         }
2066
2067 out:
2068         dwc->gadget_driver      = NULL;
2069         spin_unlock_irqrestore(&dwc->lock, flags);
2070
2071         free_irq(dwc->irq_gadget, dwc->ev_buf);
2072         free_irq(dwc->irq_wakeup, dwc);
2073
2074         return 0;
2075 }
2076
2077 static void dwc3_gadget_set_speed(struct usb_gadget *g,
2078                                   enum usb_device_speed speed)
2079 {
2080         struct dwc3             *dwc = gadget_to_dwc(g);
2081         unsigned long           flags;
2082         u32                     reg;
2083
2084         spin_lock_irqsave(&dwc->lock, flags);
2085         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2086         reg &= ~(DWC3_DCFG_SPEED_MASK);
2087
2088         /*
2089          * WORKAROUND: DWC3 revision < 2.20a have an issue
2090          * which would cause metastability state on Run/Stop
2091          * bit if we try to force the IP to USB2-only mode.
2092          *
2093          * Because of that, we cannot configure the IP to any
2094          * speed other than the SuperSpeed
2095          *
2096          * Refers to:
2097          *
2098          * STAR#9000525659: Clock Domain Crossing on DCTL in
2099          * USB 2.0 Mode
2100          */
2101         if (dwc->revision < DWC3_REVISION_220A &&
2102             !dwc->dis_metastability_quirk) {
2103                 reg |= DWC3_DCFG_SUPERSPEED;
2104         } else {
2105                 switch (speed) {
2106                 case USB_SPEED_LOW:
2107                         reg |= DWC3_DCFG_LOWSPEED;
2108                         break;
2109                 case USB_SPEED_FULL:
2110                         reg |= DWC3_DCFG_FULLSPEED;
2111                         break;
2112                 case USB_SPEED_HIGH:
2113                         reg |= DWC3_DCFG_HIGHSPEED;
2114                         break;
2115                 case USB_SPEED_SUPER:
2116                         reg |= DWC3_DCFG_SUPERSPEED;
2117                         break;
2118                 case USB_SPEED_SUPER_PLUS:
2119                         if (dwc3_is_usb31(dwc))
2120                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2121                         else
2122                                 reg |= DWC3_DCFG_SUPERSPEED;
2123                         break;
2124                 default:
2125                         dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2126
2127                         if (dwc->revision & DWC3_REVISION_IS_DWC31)
2128                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2129                         else
2130                                 reg |= DWC3_DCFG_SUPERSPEED;
2131                 }
2132         }
2133         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2134
2135         spin_unlock_irqrestore(&dwc->lock, flags);
2136 }
2137
2138 static const struct usb_gadget_ops dwc3_gadget_ops = {
2139         .get_frame              = dwc3_gadget_get_frame,
2140         .wakeup                 = dwc3_gadget_wakeup,
2141         .set_selfpowered        = dwc3_gadget_set_selfpowered,
2142         .pullup                 = dwc3_gadget_pullup,
2143         .udc_start              = dwc3_gadget_start,
2144         .udc_stop               = dwc3_gadget_stop,
2145         .udc_set_speed          = dwc3_gadget_set_speed,
2146 };
2147
2148 /* -------------------------------------------------------------------------- */
2149
2150 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
2151 {
2152         struct dwc3 *dwc = dep->dwc;
2153
2154         usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2155         dep->endpoint.maxburst = 1;
2156         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2157         if (!dep->direction)
2158                 dwc->gadget.ep0 = &dep->endpoint;
2159
2160         dep->endpoint.caps.type_control = true;
2161
2162         return 0;
2163 }
2164
2165 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
2166 {
2167         struct dwc3 *dwc = dep->dwc;
2168         int mdwidth;
2169         int kbytes;
2170         int size;
2171
2172         mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2173         /* MDWIDTH is represented in bits, we need it in bytes */
2174         mdwidth /= 8;
2175
2176         size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
2177         if (dwc3_is_usb31(dwc))
2178                 size = DWC31_GTXFIFOSIZ_TXFDEF(size);
2179         else
2180                 size = DWC3_GTXFIFOSIZ_TXFDEF(size);
2181
2182         /* FIFO Depth is in MDWDITH bytes. Multiply */
2183         size *= mdwidth;
2184
2185         kbytes = size / 1024;
2186         if (kbytes == 0)
2187                 kbytes = 1;
2188
2189         /*
2190          * FIFO sizes account an extra MDWIDTH * (kbytes + 1) bytes for
2191          * internal overhead. We don't really know how these are used,
2192          * but documentation say it exists.
2193          */
2194         size -= mdwidth * (kbytes + 1);
2195         size /= kbytes;
2196
2197         usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2198
2199         dep->endpoint.max_streams = 15;
2200         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2201         list_add_tail(&dep->endpoint.ep_list,
2202                         &dwc->gadget.ep_list);
2203         dep->endpoint.caps.type_iso = true;
2204         dep->endpoint.caps.type_bulk = true;
2205         dep->endpoint.caps.type_int = true;
2206
2207         return dwc3_alloc_trb_pool(dep);
2208 }
2209
2210 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
2211 {
2212         struct dwc3 *dwc = dep->dwc;
2213
2214         usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
2215         dep->endpoint.max_streams = 15;
2216         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2217         list_add_tail(&dep->endpoint.ep_list,
2218                         &dwc->gadget.ep_list);
2219         dep->endpoint.caps.type_iso = true;
2220         dep->endpoint.caps.type_bulk = true;
2221         dep->endpoint.caps.type_int = true;
2222
2223         return dwc3_alloc_trb_pool(dep);
2224 }
2225
2226 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
2227 {
2228         struct dwc3_ep                  *dep;
2229         bool                            direction = epnum & 1;
2230         int                             ret;
2231         u8                              num = epnum >> 1;
2232
2233         dep = kzalloc(sizeof(*dep), GFP_KERNEL);
2234         if (!dep)
2235                 return -ENOMEM;
2236
2237         dep->dwc = dwc;
2238         dep->number = epnum;
2239         dep->direction = direction;
2240         dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
2241         dwc->eps[epnum] = dep;
2242
2243         snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
2244                         direction ? "in" : "out");
2245
2246         dep->endpoint.name = dep->name;
2247
2248         if (!(dep->number > 1)) {
2249                 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2250                 dep->endpoint.comp_desc = NULL;
2251         }
2252
2253         spin_lock_init(&dep->lock);
2254
2255         if (num == 0)
2256                 ret = dwc3_gadget_init_control_endpoint(dep);
2257         else if (direction)
2258                 ret = dwc3_gadget_init_in_endpoint(dep);
2259         else
2260                 ret = dwc3_gadget_init_out_endpoint(dep);
2261
2262         if (ret)
2263                 return ret;
2264
2265         dep->endpoint.caps.dir_in = direction;
2266         dep->endpoint.caps.dir_out = !direction;
2267
2268         INIT_LIST_HEAD(&dep->pending_list);
2269         INIT_LIST_HEAD(&dep->started_list);
2270
2271         return 0;
2272 }
2273
2274 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
2275 {
2276         u8                              epnum;
2277
2278         INIT_LIST_HEAD(&dwc->gadget.ep_list);
2279
2280         for (epnum = 0; epnum < total; epnum++) {
2281                 int                     ret;
2282
2283                 ret = dwc3_gadget_init_endpoint(dwc, epnum);
2284                 if (ret)
2285                         return ret;
2286         }
2287
2288         return 0;
2289 }
2290
2291 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
2292 {
2293         struct dwc3_ep                  *dep;
2294         u8                              epnum;
2295
2296         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2297                 dep = dwc->eps[epnum];
2298                 if (!dep)
2299                         continue;
2300                 /*
2301                  * Physical endpoints 0 and 1 are special; they form the
2302                  * bi-directional USB endpoint 0.
2303                  *
2304                  * For those two physical endpoints, we don't allocate a TRB
2305                  * pool nor do we add them the endpoints list. Due to that, we
2306                  * shouldn't do these two operations otherwise we would end up
2307                  * with all sorts of bugs when removing dwc3.ko.
2308                  */
2309                 if (epnum != 0 && epnum != 1) {
2310                         dwc3_free_trb_pool(dep);
2311                         list_del(&dep->endpoint.ep_list);
2312                 }
2313
2314                 kfree(dep);
2315         }
2316 }
2317
2318 /* -------------------------------------------------------------------------- */
2319
2320 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
2321                 struct dwc3_request *req, struct dwc3_trb *trb,
2322                 const struct dwc3_event_depevt *event, int status, int chain)
2323 {
2324         unsigned int            count;
2325
2326         dwc3_ep_inc_deq(dep);
2327
2328         trace_dwc3_complete_trb(dep, trb);
2329
2330         /*
2331          * If we're in the middle of series of chained TRBs and we
2332          * receive a short transfer along the way, DWC3 will skip
2333          * through all TRBs including the last TRB in the chain (the
2334          * where CHN bit is zero. DWC3 will also avoid clearing HWO
2335          * bit and SW has to do it manually.
2336          *
2337          * We're going to do that here to avoid problems of HW trying
2338          * to use bogus TRBs for transfers.
2339          */
2340         if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
2341                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2342
2343         /*
2344          * If we're dealing with unaligned size OUT transfer, we will be left
2345          * with one TRB pending in the ring. We need to manually clear HWO bit
2346          * from that TRB.
2347          */
2348         if ((req->zero || req->unaligned) && (trb->ctrl & DWC3_TRB_CTRL_HWO)) {
2349                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2350                 return 1;
2351         }
2352
2353         count = trb->size & DWC3_TRB_SIZE_MASK;
2354         req->remaining += count;
2355
2356         if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
2357                 return 1;
2358
2359         if (event->status & DEPEVT_STATUS_SHORT && !chain)
2360                 return 1;
2361
2362         if ((event->status & DEPEVT_STATUS_IOC) &&
2363             (trb->ctrl & DWC3_TRB_CTRL_IOC))
2364                 return 1;
2365
2366         if ((event->status & DEPEVT_STATUS_LST) &&
2367             (trb->ctrl & DWC3_TRB_CTRL_LST))
2368                 return 1;
2369
2370         return 0;
2371 }
2372
2373 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
2374                 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2375                 int status)
2376 {
2377         struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2378         struct scatterlist *sg = req->sg;
2379         struct scatterlist *s;
2380         unsigned int pending = req->num_pending_sgs;
2381         unsigned int i;
2382         int ret = 0;
2383
2384         for_each_sg(sg, s, pending, i) {
2385                 trb = &dep->trb_pool[dep->trb_dequeue];
2386
2387                 if (trb->ctrl & DWC3_TRB_CTRL_HWO)
2388                         break;
2389
2390                 req->sg = sg_next(s);
2391                 req->num_pending_sgs--;
2392
2393                 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
2394                                 trb, event, status, true);
2395                 if (ret)
2396                         break;
2397         }
2398
2399         return ret;
2400 }
2401
2402 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
2403                 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2404                 int status)
2405 {
2406         struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2407
2408         return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
2409                         event, status, false);
2410 }
2411
2412 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
2413 {
2414         return req->request.actual == req->request.length;
2415 }
2416
2417 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
2418                 const struct dwc3_event_depevt *event,
2419                 struct dwc3_request *req, int status)
2420 {
2421         int ret;
2422
2423         if (req->num_pending_sgs)
2424                 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
2425                                 status);
2426         else
2427                 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2428                                 status);
2429
2430         if (req->unaligned || req->zero) {
2431                 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2432                                 status);
2433                 req->unaligned = false;
2434                 req->zero = false;
2435         }
2436
2437         req->request.actual = req->request.length - req->remaining;
2438
2439         if ((!dwc3_gadget_ep_request_completed(req) &&
2440              req->num_pending_sgs) || req->num_pending_sgs) {
2441                 if (!(event->status &
2442                         (DEPEVT_STATUS_SHORT | DEPEVT_STATUS_LST))) {
2443                         __dwc3_gadget_kick_transfer(dep);
2444                         goto out;
2445                 }
2446         }
2447
2448         dwc3_gadget_giveback(dep, req, status);
2449
2450 out:
2451         return ret;
2452 }
2453
2454 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
2455                 const struct dwc3_event_depevt *event, int status)
2456 {
2457         struct dwc3_request     *req;
2458         struct dwc3_request     *tmp;
2459
2460         list_for_each_entry_safe(req, tmp, &dep->started_list, list) {
2461                 int ret;
2462
2463                 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
2464                                 req, status);
2465                 if (ret)
2466                         break;
2467         }
2468 }
2469
2470 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
2471                 const struct dwc3_event_depevt *event)
2472 {
2473         dep->frame_number = event->parameters;
2474 }
2475
2476 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
2477                 const struct dwc3_event_depevt *event)
2478 {
2479         struct dwc3             *dwc = dep->dwc;
2480         unsigned                status = 0;
2481         bool                    stop = false;
2482
2483         dwc3_gadget_endpoint_frame_from_event(dep, event);
2484
2485         if (event->status & DEPEVT_STATUS_BUSERR)
2486                 status = -ECONNRESET;
2487
2488         if ((event->status & DEPEVT_STATUS_MISSED_ISOC) &&
2489             usb_endpoint_xfer_isoc(dep->endpoint.desc))
2490                 status = -EXDEV;
2491
2492         dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
2493
2494         if (dep->stream_capable && !list_empty(&dep->started_list))
2495                 __dwc3_gadget_kick_transfer(dep);
2496
2497         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2498             list_empty(&dep->started_list))
2499                 stop = true;
2500
2501         if (stop) {
2502                 dwc3_stop_active_transfer(dep, true);
2503                 dep->flags = DWC3_EP_ENABLED;
2504         }
2505
2506         /*
2507          * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2508          * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2509          */
2510         if (dwc->revision < DWC3_REVISION_183A) {
2511                 u32             reg;
2512                 int             i;
2513
2514                 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
2515                         dep = dwc->eps[i];
2516
2517                         if (!(dep->flags & DWC3_EP_ENABLED))
2518                                 continue;
2519
2520                         if (!list_empty(&dep->started_list))
2521                                 return;
2522                 }
2523
2524                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2525                 reg |= dwc->u1u2;
2526                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2527
2528                 dwc->u1u2 = 0;
2529         }
2530 }
2531
2532 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
2533                 const struct dwc3_event_depevt *event)
2534 {
2535         dwc3_gadget_endpoint_frame_from_event(dep, event);
2536         __dwc3_gadget_start_isoc(dep);
2537 }
2538
2539 static void dwc3_endpoint_stream_event(struct dwc3 *dwc,
2540                                        const struct dwc3_event_depevt *event)
2541 {
2542         struct dwc3_ep          *dep;
2543         struct dwc3_request     *req;
2544         u8                      epnum = event->endpoint_number;
2545         u8                      stream_id;
2546
2547         dep = dwc->eps[epnum];
2548
2549         stream_id = event->parameters;
2550
2551         /* Check for request matching the streamid and delete the timer */
2552         list_for_each_entry(req, &dep->started_list, list) {
2553                 if (req->request.stream_id == stream_id) {
2554                         if (timer_pending(&req->stream_timeout_timer))
2555                                 del_timer(&req->stream_timeout_timer);
2556                         break;
2557                 }
2558         }
2559 }
2560
2561 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2562                 const struct dwc3_event_depevt *event)
2563 {
2564         struct dwc3_ep          *dep;
2565         u8                      epnum = event->endpoint_number;
2566         u8                      cmd;
2567
2568         dep = dwc->eps[epnum];
2569
2570         if (!(dep->flags & DWC3_EP_ENABLED)) {
2571                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
2572                         return;
2573
2574                 /* Handle only EPCMDCMPLT when EP disabled */
2575                 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
2576                         return;
2577         }
2578
2579         if (epnum == 0 || epnum == 1) {
2580                 dwc3_ep0_interrupt(dwc, event);
2581                 return;
2582         }
2583
2584         switch (event->endpoint_event) {
2585         case DWC3_DEPEVT_XFERCOMPLETE:
2586                 if (!dep->stream_capable)
2587                         break;
2588                 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
2589                 /* Fall Through */
2590         case DWC3_DEPEVT_XFERINPROGRESS:
2591                 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
2592                 break;
2593         case DWC3_DEPEVT_XFERNOTREADY:
2594                 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
2595                 break;
2596         case DWC3_DEPEVT_STREAMEVT:
2597                 if (event->status == DEPEVT_STREAMEVT_FOUND)
2598                         dwc3_endpoint_stream_event(dwc, event);
2599                 break;
2600         case DWC3_DEPEVT_EPCMDCMPLT:
2601                 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
2602
2603                 if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
2604                         dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
2605                         wake_up(&dep->wait_end_transfer);
2606                 }
2607                 break;
2608         case DWC3_DEPEVT_RXTXFIFOEVT:
2609                 break;
2610         }
2611 }
2612
2613 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2614 {
2615         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2616                 spin_unlock(&dwc->lock);
2617                 dwc->gadget_driver->disconnect(&dwc->gadget);
2618                 spin_lock(&dwc->lock);
2619         }
2620 }
2621
2622 static void dwc3_suspend_gadget(struct dwc3 *dwc)
2623 {
2624         if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
2625                 spin_unlock(&dwc->lock);
2626                 dwc->gadget_driver->suspend(&dwc->gadget);
2627                 spin_lock(&dwc->lock);
2628         }
2629 }
2630
2631 static void dwc3_resume_gadget(struct dwc3 *dwc)
2632 {
2633         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2634                 spin_unlock(&dwc->lock);
2635                 dwc->gadget_driver->resume(&dwc->gadget);
2636                 spin_lock(&dwc->lock);
2637         }
2638 }
2639
2640 static void dwc3_reset_gadget(struct dwc3 *dwc)
2641 {
2642         if (!dwc->gadget_driver)
2643                 return;
2644
2645         if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2646                 spin_unlock(&dwc->lock);
2647                 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
2648                 spin_lock(&dwc->lock);
2649         }
2650 }
2651
2652 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force)
2653 {
2654         struct dwc3 *dwc = dep->dwc;
2655         struct dwc3_gadget_ep_cmd_params params;
2656         u32 cmd;
2657         int ret;
2658
2659         if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
2660             !dep->resource_index)
2661                 return;
2662
2663         /*
2664          * NOTICE: We are violating what the Databook says about the
2665          * EndTransfer command. Ideally we would _always_ wait for the
2666          * EndTransfer Command Completion IRQ, but that's causing too
2667          * much trouble synchronizing between us and gadget driver.
2668          *
2669          * We have discussed this with the IP Provider and it was
2670          * suggested to giveback all requests here, but give HW some
2671          * extra time to synchronize with the interconnect. We're using
2672          * an arbitrary 100us delay for that.
2673          *
2674          * Note also that a similar handling was tested by Synopsys
2675          * (thanks a lot Paul) and nothing bad has come out of it.
2676          * In short, what we're doing is:
2677          *
2678          * - Issue EndTransfer WITH CMDIOC bit set
2679          * - Wait 100us
2680          *
2681          * As of IP version 3.10a of the DWC_usb3 IP, the controller
2682          * supports a mode to work around the above limitation. The
2683          * software can poll the CMDACT bit in the DEPCMD register
2684          * after issuing a EndTransfer command. This mode is enabled
2685          * by writing GUCTL2[14]. This polling is already done in the
2686          * dwc3_send_gadget_ep_cmd() function so if the mode is
2687          * enabled, the EndTransfer command will have completed upon
2688          * returning from this function and we don't need to delay for
2689          * 100us.
2690          *
2691          * This mode is NOT available on the DWC_usb31 IP.
2692          */
2693
2694         cmd = DWC3_DEPCMD_ENDTRANSFER;
2695         cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2696         cmd |= DWC3_DEPCMD_CMDIOC;
2697         cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2698         memset(&params, 0, sizeof(params));
2699         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
2700         WARN_ON_ONCE(ret);
2701         dep->resource_index = 0;
2702
2703         /*
2704          * when transfer is stopped with force rm bit false, it can be
2705          * restarted by passing resource_index in params; don't loose it
2706          */
2707         if (force)
2708                 dep->resource_index = 0;
2709
2710         if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A) {
2711                 /*
2712                  * CMD COMPLETE interrupt is not getting generated for isoc
2713                  * endpoints, so don't set DWC3_EP_END_TRANSFER_PENDING flag
2714                  */
2715                 if (!usb_endpoint_xfer_isoc(dep->endpoint.desc))
2716                         dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
2717
2718                 udelay(100);
2719         }
2720 }
2721
2722 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2723 {
2724         u32 epnum;
2725
2726         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2727                 struct dwc3_ep *dep;
2728                 int ret;
2729
2730                 dep = dwc->eps[epnum];
2731                 if (!dep)
2732                         continue;
2733
2734                 if (!(dep->flags & DWC3_EP_STALL))
2735                         continue;
2736
2737                 dep->flags &= ~DWC3_EP_STALL;
2738
2739                 ret = dwc3_send_clear_stall_ep_cmd(dep);
2740                 WARN_ON_ONCE(ret);
2741         }
2742 }
2743
2744 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2745 {
2746         int                     reg;
2747
2748         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2749         reg &= ~DWC3_DCTL_INITU1ENA;
2750         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2751
2752         reg &= ~DWC3_DCTL_INITU2ENA;
2753         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2754
2755         dwc3_disconnect_gadget(dwc);
2756
2757         /* In USB 2.0, to avoid hibernation interrupt at the time of connection
2758          * clear DWC3_DCTL_KEEP_CONNECT bit.
2759          */
2760         if (dwc->has_hibernation) {
2761                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2762                 reg &= ~DWC3_DCTL_KEEP_CONNECT;
2763                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2764         }
2765
2766         dwc->gadget.speed = USB_SPEED_UNKNOWN;
2767         dwc->setup_packet_pending = false;
2768         usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
2769
2770         dwc->connected = false;
2771 }
2772
2773 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2774 {
2775         u32                     reg;
2776
2777         dwc->connected = true;
2778
2779         /*
2780          * WORKAROUND: DWC3 revisions <1.88a have an issue which
2781          * would cause a missing Disconnect Event if there's a
2782          * pending Setup Packet in the FIFO.
2783          *
2784          * There's no suggested workaround on the official Bug
2785          * report, which states that "unless the driver/application
2786          * is doing any special handling of a disconnect event,
2787          * there is no functional issue".
2788          *
2789          * Unfortunately, it turns out that we _do_ some special
2790          * handling of a disconnect event, namely complete all
2791          * pending transfers, notify gadget driver of the
2792          * disconnection, and so on.
2793          *
2794          * Our suggested workaround is to follow the Disconnect
2795          * Event steps here, instead, based on a setup_packet_pending
2796          * flag. Such flag gets set whenever we have a SETUP_PENDING
2797          * status for EP0 TRBs and gets cleared on XferComplete for the
2798          * same endpoint.
2799          *
2800          * Refers to:
2801          *
2802          * STAR#9000466709: RTL: Device : Disconnect event not
2803          * generated if setup packet pending in FIFO
2804          */
2805         if (dwc->revision < DWC3_REVISION_188A) {
2806                 if (dwc->setup_packet_pending)
2807                         dwc3_gadget_disconnect_interrupt(dwc);
2808         }
2809
2810         dwc3_reset_gadget(dwc);
2811
2812         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2813         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2814         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2815         dwc->test_mode = false;
2816         dwc3_clear_stall_all_ep(dwc);
2817
2818         /* Reset device address to zero */
2819         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2820         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2821         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2822 }
2823
2824 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2825 {
2826         struct dwc3_ep          *dep;
2827         int                     ret;
2828         u32                     reg;
2829         u8                      speed;
2830
2831         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2832         speed = reg & DWC3_DSTS_CONNECTSPD;
2833         dwc->speed = speed;
2834
2835         /*
2836          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2837          * each time on Connect Done.
2838          *
2839          * Currently we always use the reset value. If any platform
2840          * wants to set this to a different value, we need to add a
2841          * setting and update GCTL.RAMCLKSEL here.
2842          */
2843
2844         switch (speed) {
2845         case DWC3_DSTS_SUPERSPEED_PLUS:
2846                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2847                 dwc->gadget.ep0->maxpacket = 512;
2848                 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
2849                 break;
2850         case DWC3_DSTS_SUPERSPEED:
2851                 /*
2852                  * WORKAROUND: DWC3 revisions <1.90a have an issue which
2853                  * would cause a missing USB3 Reset event.
2854                  *
2855                  * In such situations, we should force a USB3 Reset
2856                  * event by calling our dwc3_gadget_reset_interrupt()
2857                  * routine.
2858                  *
2859                  * Refers to:
2860                  *
2861                  * STAR#9000483510: RTL: SS : USB3 reset event may
2862                  * not be generated always when the link enters poll
2863                  */
2864                 if (dwc->revision < DWC3_REVISION_190A)
2865                         dwc3_gadget_reset_interrupt(dwc);
2866
2867                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2868                 dwc->gadget.ep0->maxpacket = 512;
2869                 dwc->gadget.speed = USB_SPEED_SUPER;
2870                 break;
2871         case DWC3_DSTS_HIGHSPEED:
2872                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2873                 dwc->gadget.ep0->maxpacket = 64;
2874                 dwc->gadget.speed = USB_SPEED_HIGH;
2875                 break;
2876         case DWC3_DSTS_FULLSPEED:
2877                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2878                 dwc->gadget.ep0->maxpacket = 64;
2879                 dwc->gadget.speed = USB_SPEED_FULL;
2880                 break;
2881         case DWC3_DSTS_LOWSPEED:
2882                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2883                 dwc->gadget.ep0->maxpacket = 8;
2884                 dwc->gadget.speed = USB_SPEED_LOW;
2885                 break;
2886         }
2887
2888         dwc->eps[1]->endpoint.maxpacket = dwc->gadget.ep0->maxpacket;
2889
2890         /* Enable USB2 LPM Capability */
2891
2892         if ((dwc->revision > DWC3_REVISION_194A) &&
2893             (speed != DWC3_DSTS_SUPERSPEED) &&
2894             (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
2895                 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2896                 reg |= DWC3_DCFG_LPM_CAP;
2897                 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2898
2899                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2900                 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2901
2902                 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2903
2904                 /*
2905                  * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2906                  * DCFG.LPMCap is set, core responses with an ACK and the
2907                  * BESL value in the LPM token is less than or equal to LPM
2908                  * NYET threshold.
2909                  */
2910                 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2911                                 && dwc->has_lpm_erratum,
2912                                 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
2913
2914                 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2915                         reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2916
2917                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2918         } else {
2919                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2920                 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2921                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2922         }
2923
2924         dep = dwc->eps[0];
2925         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
2926         if (ret) {
2927                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2928                 return;
2929         }
2930
2931         dep = dwc->eps[1];
2932         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
2933         if (ret) {
2934                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2935                 return;
2936         }
2937
2938         /*
2939          * In USB 2.0, to avoid hibernation interrupt at the time of connection
2940          * set DWC3_DCTL_KEEP_CONNECT bit here
2941          */
2942         if (dwc->has_hibernation) {
2943                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2944                 reg |= DWC3_DCTL_KEEP_CONNECT;
2945                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2946         }
2947
2948         /*
2949          * Configure PHY via GUSB3PIPECTLn if required.
2950          *
2951          * Update GTXFIFOSIZn
2952          *
2953          * In both cases reset values should be sufficient.
2954          */
2955 }
2956
2957 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2958 {
2959         /*
2960          * TODO take core out of low power mode when that's
2961          * implemented.
2962          */
2963
2964         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2965                 spin_unlock(&dwc->lock);
2966                 dwc->gadget_driver->resume(&dwc->gadget);
2967                 spin_lock(&dwc->lock);
2968         }
2969 }
2970
2971 static irqreturn_t wakeup_interrupt(int irq, void *_dwc)
2972 {
2973         struct dwc3 *dwc = (struct dwc3 *)_dwc;
2974
2975         spin_lock(&dwc->lock);
2976         gadget_wakeup_interrupt(dwc);
2977         spin_unlock(&dwc->lock);
2978
2979         return IRQ_HANDLED;
2980 }
2981
2982 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2983                 unsigned int evtinfo)
2984 {
2985         enum dwc3_link_state    next = evtinfo & DWC3_LINK_STATE_MASK;
2986         unsigned int            pwropt;
2987
2988         /*
2989          * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2990          * Hibernation mode enabled which would show up when device detects
2991          * host-initiated U3 exit.
2992          *
2993          * In that case, device will generate a Link State Change Interrupt
2994          * from U3 to RESUME which is only necessary if Hibernation is
2995          * configured in.
2996          *
2997          * There are no functional changes due to such spurious event and we
2998          * just need to ignore it.
2999          *
3000          * Refers to:
3001          *
3002          * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
3003          * operational mode
3004          */
3005         pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
3006         if ((dwc->revision < DWC3_REVISION_250A) &&
3007                         (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
3008                 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
3009                                 (next == DWC3_LINK_STATE_RESUME)) {
3010                         return;
3011                 }
3012         }
3013
3014         /*
3015          * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
3016          * on the link partner, the USB session might do multiple entry/exit
3017          * of low power states before a transfer takes place.
3018          *
3019          * Due to this problem, we might experience lower throughput. The
3020          * suggested workaround is to disable DCTL[12:9] bits if we're
3021          * transitioning from U1/U2 to U0 and enable those bits again
3022          * after a transfer completes and there are no pending transfers
3023          * on any of the enabled endpoints.
3024          *
3025          * This is the first half of that workaround.
3026          *
3027          * Refers to:
3028          *
3029          * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
3030          * core send LGO_Ux entering U0
3031          */
3032         if (dwc->revision < DWC3_REVISION_183A) {
3033                 if (next == DWC3_LINK_STATE_U0) {
3034                         u32     u1u2;
3035                         u32     reg;
3036
3037                         switch (dwc->link_state) {
3038                         case DWC3_LINK_STATE_U1:
3039                         case DWC3_LINK_STATE_U2:
3040                                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3041                                 u1u2 = reg & (DWC3_DCTL_INITU2ENA
3042                                                 | DWC3_DCTL_ACCEPTU2ENA
3043                                                 | DWC3_DCTL_INITU1ENA
3044                                                 | DWC3_DCTL_ACCEPTU1ENA);
3045
3046                                 if (!dwc->u1u2)
3047                                         dwc->u1u2 = reg & u1u2;
3048
3049                                 reg &= ~u1u2;
3050
3051                                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3052                                 break;
3053                         default:
3054                                 /* do nothing */
3055                                 break;
3056                         }
3057                 }
3058         }
3059
3060         switch (next) {
3061         case DWC3_LINK_STATE_U1:
3062                 if (dwc->speed == USB_SPEED_SUPER)
3063                         dwc3_suspend_gadget(dwc);
3064                 break;
3065         case DWC3_LINK_STATE_U2:
3066         case DWC3_LINK_STATE_U3:
3067                 dwc3_suspend_gadget(dwc);
3068                 break;
3069         case DWC3_LINK_STATE_RESUME:
3070                 dwc3_resume_gadget(dwc);
3071                 break;
3072         default:
3073                 /* do nothing */
3074                 break;
3075         }
3076
3077         dwc->link_state = next;
3078 }
3079
3080 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
3081                                           unsigned int evtinfo)
3082 {
3083         enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
3084
3085         if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
3086                 dwc3_suspend_gadget(dwc);
3087
3088         dwc->link_state = next;
3089 }
3090
3091 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
3092                 unsigned int evtinfo)
3093 {
3094         unsigned int is_ss = evtinfo & BIT(4);
3095
3096         /*
3097          * WORKAROUND: DWC3 revison 2.20a with hibernation support
3098          * have a known issue which can cause USB CV TD.9.23 to fail
3099          * randomly.
3100          *
3101          * Because of this issue, core could generate bogus hibernation
3102          * events which SW needs to ignore.
3103          *
3104          * Refers to:
3105          *
3106          * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
3107          * Device Fallback from SuperSpeed
3108          */
3109         if ((!!is_ss ^ (dwc->speed >= DWC3_DSTS_SUPERSPEED)) &&
3110             (!(dwc->has_hibernation)))
3111                 return;
3112
3113         /* enter hibernation here */
3114         gadget_hibernation_interrupt(dwc);
3115 }
3116
3117 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
3118                 const struct dwc3_event_devt *event)
3119 {
3120         switch (event->type) {
3121         case DWC3_DEVICE_EVENT_DISCONNECT:
3122                 dwc3_gadget_disconnect_interrupt(dwc);
3123                 break;
3124         case DWC3_DEVICE_EVENT_RESET:
3125                 dwc3_gadget_reset_interrupt(dwc);
3126                 break;
3127         case DWC3_DEVICE_EVENT_CONNECT_DONE:
3128                 dwc3_gadget_conndone_interrupt(dwc);
3129                 break;
3130         case DWC3_DEVICE_EVENT_WAKEUP:
3131                 dwc3_gadget_wakeup_interrupt(dwc);
3132                 break;
3133         case DWC3_DEVICE_EVENT_HIBER_REQ:
3134                 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
3135                                         "unexpected hibernation event\n"))
3136                         break;
3137
3138                 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
3139                 break;
3140         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
3141                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
3142                 break;
3143         case DWC3_DEVICE_EVENT_EOPF:
3144                 /* It changed to be suspend event for version 2.30a and above */
3145                 if (dwc->revision >= DWC3_REVISION_230A) {
3146                         /*
3147                          * Ignore suspend event until the gadget enters into
3148                          * USB_STATE_CONFIGURED state.
3149                          */
3150                         if (dwc->gadget.state >= USB_STATE_CONFIGURED)
3151                                 dwc3_gadget_suspend_interrupt(dwc,
3152                                                 event->event_info);
3153                 }
3154                 break;
3155         case DWC3_DEVICE_EVENT_SOF:
3156         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
3157         case DWC3_DEVICE_EVENT_CMD_CMPL:
3158         case DWC3_DEVICE_EVENT_OVERFLOW:
3159                 break;
3160         default:
3161                 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
3162         }
3163 }
3164
3165 static void dwc3_process_event_entry(struct dwc3 *dwc,
3166                 const union dwc3_event *event)
3167 {
3168         trace_dwc3_event(event->raw, dwc);
3169
3170         if (!event->type.is_devspec)
3171                 dwc3_endpoint_interrupt(dwc, &event->depevt);
3172         else if (event->type.type == DWC3_EVENT_TYPE_DEV)
3173                 dwc3_gadget_interrupt(dwc, &event->devt);
3174         else
3175                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
3176 }
3177
3178 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
3179 {
3180         struct dwc3 *dwc = evt->dwc;
3181         irqreturn_t ret = IRQ_NONE;
3182         int left;
3183         u32 reg;
3184
3185         left = evt->count;
3186
3187         if (!(evt->flags & DWC3_EVENT_PENDING))
3188                 return IRQ_NONE;
3189
3190         while (left > 0) {
3191                 union dwc3_event event;
3192
3193                 event.raw = *(u32 *) (evt->cache + evt->lpos);
3194
3195                 dwc3_process_event_entry(dwc, &event);
3196
3197                 /*
3198                  * FIXME we wrap around correctly to the next entry as
3199                  * almost all entries are 4 bytes in size. There is one
3200                  * entry which has 12 bytes which is a regular entry
3201                  * followed by 8 bytes data. ATM I don't know how
3202                  * things are organized if we get next to the a
3203                  * boundary so I worry about that once we try to handle
3204                  * that.
3205                  */
3206                 evt->lpos = (evt->lpos + 4) % evt->length;
3207                 left -= 4;
3208
3209                 if (dwc->is_hibernated)
3210                         break;
3211         }
3212
3213         evt->count = 0;
3214         evt->flags &= ~DWC3_EVENT_PENDING;
3215         ret = IRQ_HANDLED;
3216
3217         if (dwc->is_hibernated)
3218                 return ret;
3219
3220         /* Unmask interrupt */
3221         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3222         reg &= ~DWC3_GEVNTSIZ_INTMASK;
3223         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3224
3225         if (dwc->imod_interval) {
3226                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
3227                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
3228         }
3229
3230         return ret;
3231 }
3232
3233 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
3234 {
3235         struct dwc3_event_buffer *evt = _evt;
3236         struct dwc3 *dwc = evt->dwc;
3237         unsigned long flags;
3238         irqreturn_t ret = IRQ_NONE;
3239
3240         spin_lock_irqsave(&dwc->lock, flags);
3241         ret = dwc3_process_event_buf(evt);
3242         spin_unlock_irqrestore(&dwc->lock, flags);
3243
3244         return ret;
3245 }
3246
3247 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
3248 {
3249         struct dwc3 *dwc = evt->dwc;
3250         u32 amount;
3251         u32 count;
3252         u32 reg;
3253
3254         if (pm_runtime_suspended(dwc->dev)) {
3255                 pm_runtime_get(dwc->dev);
3256                 disable_irq_nosync(dwc->irq_gadget);
3257                 dwc->pending_events = true;
3258                 return IRQ_HANDLED;
3259         }
3260
3261         if (dwc->is_hibernated)
3262                 return IRQ_HANDLED;
3263
3264         /*
3265          * With PCIe legacy interrupt, test shows that top-half irq handler can
3266          * be called again after HW interrupt deassertion. Check if bottom-half
3267          * irq event handler completes before caching new event to prevent
3268          * losing events.
3269          */
3270         if (evt->flags & DWC3_EVENT_PENDING)
3271                 return IRQ_HANDLED;
3272
3273         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
3274         count &= DWC3_GEVNTCOUNT_MASK;
3275         if (!count)
3276                 return IRQ_NONE;
3277
3278         evt->count = count;
3279         evt->flags |= DWC3_EVENT_PENDING;
3280
3281         /* Mask interrupt */
3282         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3283         reg |= DWC3_GEVNTSIZ_INTMASK;
3284         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3285
3286         amount = min(count, evt->length - evt->lpos);
3287         memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
3288
3289         if (amount < count)
3290                 memcpy(evt->cache, evt->buf, count - amount);
3291
3292         dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
3293
3294         return IRQ_WAKE_THREAD;
3295 }
3296
3297 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
3298 {
3299         struct dwc3_event_buffer        *evt = _evt;
3300
3301         return dwc3_check_event_buf(evt);
3302 }
3303
3304 static int dwc3_gadget_get_irq(struct dwc3 *dwc)
3305 {
3306         struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
3307         int irq, irq_hiber;
3308
3309         irq = platform_get_irq_byname(dwc3_pdev, "peripheral");
3310         if (irq > 0)
3311                 goto out;
3312
3313         if (irq == -EPROBE_DEFER)
3314                 goto out;
3315
3316         irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
3317         if (irq > 0)
3318                 goto out;
3319
3320         if (irq == -EPROBE_DEFER)
3321                 goto out;
3322
3323         irq = platform_get_irq(dwc3_pdev, 0);
3324         if (irq > 0)
3325                 dwc->irq_gadget = irq;
3326
3327         if (irq == -EPROBE_DEFER)
3328                 goto out;
3329 out:
3330         /* look for wakeup interrupt if hibernation is supported */
3331         if (dwc->has_hibernation) {
3332                 irq_hiber = platform_get_irq_byname(dwc3_pdev, "hiber");
3333                 if (irq_hiber > 0) {
3334                         dwc->irq_wakeup = irq_hiber;
3335                 } else {
3336                         irq_hiber = platform_get_irq(dwc3_pdev, 2);
3337                         if (irq_hiber > 0)
3338                                 dwc->irq_wakeup = irq_hiber;
3339                 }
3340         }
3341
3342         return irq;
3343 }
3344
3345 /**
3346  * dwc3_gadget_init - initializes gadget related registers
3347  * @dwc: pointer to our controller context structure
3348  *
3349  * Returns 0 on success otherwise negative errno.
3350  */
3351 int dwc3_gadget_init(struct dwc3 *dwc)
3352 {
3353         int ret;
3354         int irq;
3355
3356         irq = dwc3_gadget_get_irq(dwc);
3357         if (irq < 0) {
3358                 ret = irq;
3359                 goto err0;
3360         }
3361
3362         dwc->irq_gadget = irq;
3363
3364         dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
3365                                           sizeof(*dwc->ep0_trb) * 2,
3366                                           &dwc->ep0_trb_addr, GFP_KERNEL);
3367         if (!dwc->ep0_trb) {
3368                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
3369                 ret = -ENOMEM;
3370                 goto err0;
3371         }
3372
3373         dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
3374         if (!dwc->setup_buf) {
3375                 ret = -ENOMEM;
3376                 goto err1;
3377         }
3378
3379         dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
3380                         &dwc->bounce_addr, GFP_KERNEL);
3381         if (!dwc->bounce) {
3382                 ret = -ENOMEM;
3383                 goto err2;
3384         }
3385
3386         init_completion(&dwc->ep0_in_setup);
3387
3388         dwc->gadget.ops                 = &dwc3_gadget_ops;
3389         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
3390         dwc->gadget.sg_supported        = true;
3391         dwc->gadget.name                = "dwc3-gadget";
3392         dwc->gadget.is_otg              = dwc->dr_mode == USB_DR_MODE_OTG;
3393
3394         /*
3395          * FIXME We might be setting max_speed to <SUPER, however versions
3396          * <2.20a of dwc3 have an issue with metastability (documented
3397          * elsewhere in this driver) which tells us we can't set max speed to
3398          * anything lower than SUPER.
3399          *
3400          * Because gadget.max_speed is only used by composite.c and function
3401          * drivers (i.e. it won't go into dwc3's registers) we are allowing this
3402          * to happen so we avoid sending SuperSpeed Capability descriptor
3403          * together with our BOS descriptor as that could confuse host into
3404          * thinking we can handle super speed.
3405          *
3406          * Note that, in fact, we won't even support GetBOS requests when speed
3407          * is less than super speed because we don't have means, yet, to tell
3408          * composite.c that we are USB 2.0 + LPM ECN.
3409          */
3410         if (dwc->revision < DWC3_REVISION_220A &&
3411             !dwc->dis_metastability_quirk)
3412                 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
3413                                 dwc->revision);
3414
3415         dwc->gadget.max_speed           = dwc->maximum_speed;
3416
3417         /*
3418          * REVISIT: Here we should clear all pending IRQs to be
3419          * sure we're starting from a well known location.
3420          */
3421
3422         ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
3423         if (ret)
3424                 goto err3;
3425
3426         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
3427         if (ret) {
3428                 dev_err(dwc->dev, "failed to register udc\n");
3429                 goto err4;
3430         }
3431
3432         if (dwc->dr_mode == USB_DR_MODE_OTG) {
3433                 struct usb_phy *phy;
3434
3435                 phy = usb_get_phy(USB_PHY_TYPE_USB3);
3436                 if (!IS_ERR(phy)) {
3437                         if (phy && phy->otg) {
3438                                 ret = otg_set_peripheral(phy->otg,
3439                                                          &dwc->gadget);
3440                                 if (ret) {
3441                                         dev_err(dwc->dev,
3442                                                 "otg_set_peripheral failed\n");
3443                                         usb_put_phy(phy);
3444                                         phy = NULL;
3445                                         goto err4;
3446                                 }
3447                         } else {
3448                                 usb_put_phy(phy);
3449                                 phy = NULL;
3450                         }
3451                 }
3452         }
3453
3454         return 0;
3455
3456 err4:
3457         dwc3_gadget_free_endpoints(dwc);
3458
3459 err3:
3460         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3461                         dwc->bounce_addr);
3462
3463 err2:
3464         kfree(dwc->setup_buf);
3465
3466 err1:
3467         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3468                         dwc->ep0_trb, dwc->ep0_trb_addr);
3469
3470 err0:
3471         return ret;
3472 }
3473
3474 /* -------------------------------------------------------------------------- */
3475
3476 void dwc3_gadget_exit(struct dwc3 *dwc)
3477 {
3478         usb_del_gadget_udc(&dwc->gadget);
3479         dwc3_gadget_free_endpoints(dwc);
3480         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3481                           dwc->bounce_addr);
3482         kfree(dwc->setup_buf);
3483         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3484                           dwc->ep0_trb, dwc->ep0_trb_addr);
3485 }
3486
3487 int dwc3_gadget_suspend(struct dwc3 *dwc)
3488 {
3489         if (!dwc->gadget_driver)
3490                 return 0;
3491
3492         if (dwc->is_hibernated) {
3493                 /*
3494                  * As we are about to suspend, wake the controller from
3495                  * D3 & hibernation states
3496                  */
3497                 dwc->force_hiber_wake = true;
3498                 gadget_wakeup_interrupt(dwc);
3499                 dwc->force_hiber_wake = false;
3500         }
3501
3502         dwc3_gadget_run_stop(dwc, false, false);
3503         dwc3_disconnect_gadget(dwc);
3504         __dwc3_gadget_stop(dwc);
3505
3506         return 0;
3507 }
3508
3509 int dwc3_gadget_resume(struct dwc3 *dwc)
3510 {
3511         int                     ret;
3512         u32                     reg;
3513
3514         if (!dwc->gadget_driver)
3515                 return 0;
3516
3517         ret = __dwc3_gadget_start(dwc);
3518         if (ret < 0)
3519                 goto err0;
3520
3521         ret = dwc3_gadget_run_stop(dwc, true, false);
3522         if (ret < 0)
3523                 goto err1;
3524
3525         /* In USB 2.0, to avoid hibernation interrupt at the time of connection
3526          * set DWC3_DCTL_KEEP_CONNECT bit.
3527          */
3528         if (dwc->has_hibernation) {
3529                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3530                 reg |= DWC3_DCTL_KEEP_CONNECT;
3531                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3532         }
3533
3534         return 0;
3535
3536 err1:
3537         __dwc3_gadget_stop(dwc);
3538
3539 err0:
3540         return ret;
3541 }
3542
3543 void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
3544 {
3545         if (dwc->pending_events) {
3546                 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
3547                 dwc->pending_events = false;
3548                 enable_irq(dwc->irq_gadget);
3549         }
3550 }