drivers/infiniband/hw/ocrdma/ocrdma_verbs.c

   1 /*******************************************************************
   2  * This file is part of the Emulex RoCE Device Driver for          *
   3  * RoCE (RDMA over Converged Ethernet) adapters.                   *
   4  * Copyright (C) 2008-2012 Emulex. All rights reserved.            *
   5  * EMULEX and SLI are trademarks of Emulex.                        *
   6  * www.emulex.com                                                  *
   7  *                                                                 *
   8  * This program is free software; you can redistribute it and/or   *
   9  * modify it under the terms of version 2 of the GNU General       *
  10  * Public License as published by the Free Software Foundation.    *
  11  * This program is distributed in the hope that it will be useful. *
  12  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
  13  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
  14  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
  15  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
  16  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
  17  * more details, a copy of which can be found in the file COPYING  *
  18  * included with this package.                                     *
  19  *
  20  * Contact Information:
  21  * linux-drivers@emulex.com
  22  *
  23  * Emulex
  24  * 3333 Susan Street
  25  * Costa Mesa, CA 92626
  26  *******************************************************************/
  27
  28 #include <linux/dma-mapping.h>
  29 #include <rdma/ib_verbs.h>
  30 #include <rdma/ib_user_verbs.h>
  31 #include <rdma/iw_cm.h>
  32 #include <rdma/ib_umem.h>
  33 #include <rdma/ib_addr.h>
  34
  35 #include "ocrdma.h"
  36 #include "ocrdma_hw.h"
  37 #include "ocrdma_verbs.h"
  38 #include "ocrdma_abi.h"
  39
  40 int ocrdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
  41 {
  42         if (index > 1)
  43                 return -EINVAL;
  44
  45         *pkey = 0xffff;
  46         return 0;
  47 }
  48
  49 int ocrdma_query_gid(struct ib_device *ibdev, u8 port,
  50                      int index, union ib_gid *sgid)
  51 {
  52         struct ocrdma_dev *dev;
  53
  54         dev = get_ocrdma_dev(ibdev);
  55         memset(sgid, 0, sizeof(*sgid));
  56         if (index >= OCRDMA_MAX_SGID)
  57                 return -EINVAL;
  58
  59         memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid));
  60
  61         return 0;
  62 }
  63
  64 int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr)
  65 {
  66         struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
  67
  68         memset(attr, 0, sizeof *attr);
  69         memcpy(&attr->fw_ver, &dev->attr.fw_ver[0],
  70                min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver)));
  71         ocrdma_get_guid(dev, (u8 *)&attr->sys_image_guid);
  72         attr->max_mr_size = ~0ull;
  73         attr->page_size_cap = 0xffff000;
  74         attr->vendor_id = dev->nic_info.pdev->vendor;
  75         attr->vendor_part_id = dev->nic_info.pdev->device;
  76         attr->hw_ver = 0;
  77         attr->max_qp = dev->attr.max_qp;
  78         attr->max_ah = dev->attr.max_qp;
  79         attr->max_qp_wr = dev->attr.max_wqe;
  80
  81         attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
  82                                         IB_DEVICE_RC_RNR_NAK_GEN |
  83                                         IB_DEVICE_SHUTDOWN_PORT |
  84                                         IB_DEVICE_SYS_IMAGE_GUID |
  85                                         IB_DEVICE_LOCAL_DMA_LKEY;
  86         attr->max_sge = min(dev->attr.max_send_sge, dev->attr.max_srq_sge);
  87         attr->max_sge_rd = 0;
  88         attr->max_cq = dev->attr.max_cq;
  89         attr->max_cqe = dev->attr.max_cqe;
  90         attr->max_mr = dev->attr.max_mr;
  91         attr->max_mw = 0;
  92         attr->max_pd = dev->attr.max_pd;
  93         attr->atomic_cap = 0;
  94         attr->max_fmr = 0;
  95         attr->max_map_per_fmr = 0;
  96         attr->max_qp_rd_atom =
  97             min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp);
  98         attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp;
  99         attr->max_srq = (dev->attr.max_qp - 1);
 100         attr->max_srq_sge = dev->attr.max_srq_sge;
 101         attr->max_srq_wr = dev->attr.max_rqe;
 102         attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay;
 103         attr->max_fast_reg_page_list_len = 0;
 104         attr->max_pkeys = 1;
 105         return 0;
 106 }
 107
 108 int ocrdma_query_port(struct ib_device *ibdev,
 109                       u8 port, struct ib_port_attr *props)
 110 {
 111         enum ib_port_state port_state;
 112         struct ocrdma_dev *dev;
 113         struct net_device *netdev;
 114
 115         dev = get_ocrdma_dev(ibdev);
 116         if (port > 1) {
 117                 pr_err("%s(%d) invalid_port=0x%x\n", __func__,
 118                        dev->id, port);
 119                 return -EINVAL;
 120         }
 121         netdev = dev->nic_info.netdev;
 122         if (netif_running(netdev) && netif_oper_up(netdev)) {
 123                 port_state = IB_PORT_ACTIVE;
 124                 props->phys_state = 5;
 125         } else {
 126                 port_state = IB_PORT_DOWN;
 127                 props->phys_state = 3;
 128         }
 129         props->max_mtu = IB_MTU_4096;
 130         props->active_mtu = iboe_get_mtu(netdev->mtu);
 131         props->lid = 0;
 132         props->lmc = 0;
 133         props->sm_lid = 0;
 134         props->sm_sl = 0;
 135         props->state = port_state;
 136         props->port_cap_flags =
 137             IB_PORT_CM_SUP |
 138             IB_PORT_REINIT_SUP |
 139             IB_PORT_DEVICE_MGMT_SUP | IB_PORT_VENDOR_CLASS_SUP;
 140         props->gid_tbl_len = OCRDMA_MAX_SGID;
 141         props->pkey_tbl_len = 1;
 142         props->bad_pkey_cntr = 0;
 143         props->qkey_viol_cntr = 0;
 144         props->active_width = IB_WIDTH_1X;
 145         props->active_speed = 4;
 146         props->max_msg_sz = 0x80000000;
 147         props->max_vl_num = 4;
 148         return 0;
 149 }
 150
 151 int ocrdma_modify_port(struct ib_device *ibdev, u8 port, int mask,
 152                        struct ib_port_modify *props)
 153 {
 154         struct ocrdma_dev *dev;
 155
 156         dev = get_ocrdma_dev(ibdev);
 157         if (port > 1) {
 158                 pr_err("%s(%d) invalid_port=0x%x\n", __func__, dev->id, port);
 159                 return -EINVAL;
 160         }
 161         return 0;
 162 }
 163
 164 static int ocrdma_add_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
 165                            unsigned long len)
 166 {
 167         struct ocrdma_mm *mm;
 168
 169         mm = kzalloc(sizeof(*mm), GFP_KERNEL);
 170         if (mm == NULL)
 171                 return -ENOMEM;
 172         mm->key.phy_addr = phy_addr;
 173         mm->key.len = len;
 174         INIT_LIST_HEAD(&mm->entry);
 175
 176         mutex_lock(&uctx->mm_list_lock);
 177         list_add_tail(&mm->entry, &uctx->mm_head);
 178         mutex_unlock(&uctx->mm_list_lock);
 179         return 0;
 180 }
 181
 182 static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
 183                             unsigned long len)
 184 {
 185         struct ocrdma_mm *mm, *tmp;
 186
 187         mutex_lock(&uctx->mm_list_lock);
 188         list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
 189                 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
 190                         continue;
 191
 192                 list_del(&mm->entry);
 193                 kfree(mm);
 194                 break;
 195         }
 196         mutex_unlock(&uctx->mm_list_lock);
 197 }
 198
 199 static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
 200                               unsigned long len)
 201 {
 202         bool found = false;
 203         struct ocrdma_mm *mm;
 204
 205         mutex_lock(&uctx->mm_list_lock);
 206         list_for_each_entry(mm, &uctx->mm_head, entry) {
 207                 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
 208                         continue;
 209
 210                 found = true;
 211                 break;
 212         }
 213         mutex_unlock(&uctx->mm_list_lock);
 214         return found;
 215 }
 216
 217 struct ib_ucontext *ocrdma_alloc_ucontext(struct ib_device *ibdev,
 218                                           struct ib_udata *udata)
 219 {
 220         int status;
 221         struct ocrdma_ucontext *ctx;
 222         struct ocrdma_alloc_ucontext_resp resp;
 223         struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
 224         struct pci_dev *pdev = dev->nic_info.pdev;
 225         u32 map_len = roundup(sizeof(u32) * 2048, PAGE_SIZE);
 226
 227         if (!udata)
 228                 return ERR_PTR(-EFAULT);
 229         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 230         if (!ctx)
 231                 return ERR_PTR(-ENOMEM);
 232         ctx->dev = dev;
 233         INIT_LIST_HEAD(&ctx->mm_head);
 234         mutex_init(&ctx->mm_list_lock);
 235
 236         ctx->ah_tbl.va = dma_alloc_coherent(&pdev->dev, map_len,
 237                                             &ctx->ah_tbl.pa, GFP_KERNEL);
 238         if (!ctx->ah_tbl.va) {
 239                 kfree(ctx);
 240                 return ERR_PTR(-ENOMEM);
 241         }
 242         memset(ctx->ah_tbl.va, 0, map_len);
 243         ctx->ah_tbl.len = map_len;
 244
 245         resp.ah_tbl_len = ctx->ah_tbl.len;
 246         resp.ah_tbl_page = ctx->ah_tbl.pa;
 247
 248         status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len);
 249         if (status)
 250                 goto map_err;
 251         resp.dev_id = dev->id;
 252         resp.max_inline_data = dev->attr.max_inline_data;
 253         resp.wqe_size = dev->attr.wqe_size;
 254         resp.rqe_size = dev->attr.rqe_size;
 255         resp.dpp_wqe_size = dev->attr.wqe_size;
 256         resp.rsvd = 0;
 257
 258         memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver));
 259         status = ib_copy_to_udata(udata, &resp, sizeof(resp));
 260         if (status)
 261                 goto cpy_err;
 262         return &ctx->ibucontext;
 263
 264 cpy_err:
 265         ocrdma_del_mmap(ctx, ctx->ah_tbl.pa, ctx->ah_tbl.len);
 266 map_err:
 267         dma_free_coherent(&pdev->dev, ctx->ah_tbl.len, ctx->ah_tbl.va,
 268                           ctx->ah_tbl.pa);
 269         kfree(ctx);
 270         return ERR_PTR(status);
 271 }
 272
 273 int ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx)
 274 {
 275         struct ocrdma_mm *mm, *tmp;
 276         struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibctx);
 277         struct pci_dev *pdev = uctx->dev->nic_info.pdev;
 278
 279         ocrdma_del_mmap(uctx, uctx->ah_tbl.pa, uctx->ah_tbl.len);
 280         dma_free_coherent(&pdev->dev, uctx->ah_tbl.len, uctx->ah_tbl.va,
 281                           uctx->ah_tbl.pa);
 282
 283         list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
 284                 list_del(&mm->entry);
 285                 kfree(mm);
 286         }
 287         kfree(uctx);
 288         return 0;
 289 }
 290
 291 int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 292 {
 293         struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(context);
 294         struct ocrdma_dev *dev = ucontext->dev;
 295         unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
 296         u64 unmapped_db = (u64) dev->nic_info.unmapped_db;
 297         unsigned long len = (vma->vm_end - vma->vm_start);
 298         int status = 0;
 299         bool found;
 300
 301         if (vma->vm_start & (PAGE_SIZE - 1))
 302                 return -EINVAL;
 303         found = ocrdma_search_mmap(ucontext, vma->vm_pgoff << PAGE_SHIFT, len);
 304         if (!found)
 305                 return -EINVAL;
 306
 307         if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
 308                 dev->nic_info.db_total_size)) &&
 309                 (len <= dev->nic_info.db_page_size)) {
 310                 /* doorbell mapping */
 311                 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
 312                                             len, vma->vm_page_prot);
 313         } else if (dev->nic_info.dpp_unmapped_len &&
 314                 (vm_page >= (u64) dev->nic_info.dpp_unmapped_addr) &&
 315                 (vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr +
 316                         dev->nic_info.dpp_unmapped_len)) &&
 317                 (len <= dev->nic_info.dpp_unmapped_len)) {
 318                 /* dpp area mapping */
 319                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 320                 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
 321                                             len, vma->vm_page_prot);
 322         } else {
 323                 /* queue memory mapping */
 324                 status = remap_pfn_range(vma, vma->vm_start,
 325                                          vma->vm_pgoff, len, vma->vm_page_prot);
 326         }
 327         return status;
 328 }
 329
 330 static int ocrdma_copy_pd_uresp(struct ocrdma_pd *pd,
 331                                 struct ib_ucontext *ib_ctx,
 332                                 struct ib_udata *udata)
 333 {
 334         int status;
 335         u64 db_page_addr;
 336         u64 dpp_page_addr = 0;
 337         u32 db_page_size;
 338         struct ocrdma_alloc_pd_uresp rsp;
 339         struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ib_ctx);
 340
 341         rsp.id = pd->id;
 342         rsp.dpp_enabled = pd->dpp_enabled;
 343         db_page_addr = pd->dev->nic_info.unmapped_db +
 344                         (pd->id * pd->dev->nic_info.db_page_size);
 345         db_page_size = pd->dev->nic_info.db_page_size;
 346
 347         status = ocrdma_add_mmap(uctx, db_page_addr, db_page_size);
 348         if (status)
 349                 return status;
 350
 351         if (pd->dpp_enabled) {
 352                 dpp_page_addr = pd->dev->nic_info.dpp_unmapped_addr +
 353                                 (pd->id * OCRDMA_DPP_PAGE_SIZE);
 354                 status = ocrdma_add_mmap(uctx, dpp_page_addr,
 355                                  OCRDMA_DPP_PAGE_SIZE);
 356                 if (status)
 357                         goto dpp_map_err;
 358                 rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr);
 359                 rsp.dpp_page_addr_lo = dpp_page_addr;
 360         }
 361
 362         status = ib_copy_to_udata(udata, &rsp, sizeof(rsp));
 363         if (status)
 364                 goto ucopy_err;
 365
 366         pd->uctx = uctx;
 367         return 0;
 368
 369 ucopy_err:
 370         if (pd->dpp_enabled)
 371                 ocrdma_del_mmap(pd->uctx, dpp_page_addr, OCRDMA_DPP_PAGE_SIZE);
 372 dpp_map_err:
 373         ocrdma_del_mmap(pd->uctx, db_page_addr, db_page_size);
 374         return status;
 375 }
 376
 377 struct ib_pd *ocrdma_alloc_pd(struct ib_device *ibdev,
 378                               struct ib_ucontext *context,
 379                               struct ib_udata *udata)
 380 {
 381         struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
 382         struct ocrdma_pd *pd;
 383         int status;
 384
 385         pd = kzalloc(sizeof(*pd), GFP_KERNEL);
 386         if (!pd)
 387                 return ERR_PTR(-ENOMEM);
 388         pd->dev = dev;
 389         if (udata && context) {
 390                 pd->dpp_enabled = (dev->nic_info.dev_family ==
 391                                         OCRDMA_GEN2_FAMILY) ? true : false;
 392                 pd->num_dpp_qp =
 393                         pd->dpp_enabled ? OCRDMA_PD_MAX_DPP_ENABLED_QP : 0;
 394         }
 395         status = ocrdma_mbx_alloc_pd(dev, pd);
 396         if (status) {
 397                 kfree(pd);
 398                 return ERR_PTR(status);
 399         }
 400
 401         if (udata && context) {
 402                 status = ocrdma_copy_pd_uresp(pd, context, udata);
 403                 if (status)
 404                         goto err;
 405         }
 406         return &pd->ibpd;
 407
 408 err:
 409         ocrdma_dealloc_pd(&pd->ibpd);
 410         return ERR_PTR(status);
 411 }
 412
 413 int ocrdma_dealloc_pd(struct ib_pd *ibpd)
 414 {
 415         struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
 416         struct ocrdma_dev *dev = pd->dev;
 417         int status;
 418         u64 usr_db;
 419
 420         status = ocrdma_mbx_dealloc_pd(dev, pd);
 421         if (pd->uctx) {
 422                 u64 dpp_db = dev->nic_info.dpp_unmapped_addr +
 423                     (pd->id * OCRDMA_DPP_PAGE_SIZE);
 424                 if (pd->dpp_enabled)
 425                         ocrdma_del_mmap(pd->uctx, dpp_db, OCRDMA_DPP_PAGE_SIZE);
 426                 usr_db = dev->nic_info.unmapped_db +
 427                     (pd->id * dev->nic_info.db_page_size);
 428                 ocrdma_del_mmap(pd->uctx, usr_db, dev->nic_info.db_page_size);
 429         }
 430         kfree(pd);
 431         return status;
 432 }
 433
 434 static struct ocrdma_mr *ocrdma_alloc_lkey(struct ib_pd *ibpd,
 435                                            int acc, u32 num_pbls,
 436                                            u32 addr_check)
 437 {
 438         int status;
 439         struct ocrdma_mr *mr;
 440         struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
 441         struct ocrdma_dev *dev = pd->dev;
 442
 443         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
 444                 pr_err("%s(%d) leaving err, invalid access rights\n",
 445                        __func__, dev->id);
 446                 return ERR_PTR(-EINVAL);
 447         }
 448
 449         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 450         if (!mr)
 451                 return ERR_PTR(-ENOMEM);
 452         mr->hwmr.dev = dev;
 453         mr->hwmr.fr_mr = 0;
 454         mr->hwmr.local_rd = 1;
 455         mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
 456         mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
 457         mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
 458         mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0;
 459         mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
 460         mr->hwmr.num_pbls = num_pbls;
 461
 462         status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pd->id, addr_check);
 463         if (status) {
 464                 kfree(mr);
 465                 return ERR_PTR(-ENOMEM);
 466         }
 467         mr->pd = pd;
 468         mr->ibmr.lkey = mr->hwmr.lkey;
 469         if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
 470                 mr->ibmr.rkey = mr->hwmr.lkey;
 471         return mr;
 472 }
 473
 474 struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc)
 475 {
 476         struct ocrdma_mr *mr;
 477
 478         mr = ocrdma_alloc_lkey(ibpd, acc, 0, OCRDMA_ADDR_CHECK_DISABLE);
 479         if (IS_ERR(mr))
 480                 return ERR_CAST(mr);
 481
 482         return &mr->ibmr;
 483 }
 484
 485 static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev,
 486                                    struct ocrdma_hw_mr *mr)
 487 {
 488         struct pci_dev *pdev = dev->nic_info.pdev;
 489         int i = 0;
 490
 491         if (mr->pbl_table) {
 492                 for (i = 0; i < mr->num_pbls; i++) {
 493                         if (!mr->pbl_table[i].va)
 494                                 continue;
 495                         dma_free_coherent(&pdev->dev, mr->pbl_size,
 496                                           mr->pbl_table[i].va,
 497                                           mr->pbl_table[i].pa);
 498                 }
 499                 kfree(mr->pbl_table);
 500                 mr->pbl_table = NULL;
 501         }
 502 }
 503
 504 static int ocrdma_get_pbl_info(struct ocrdma_mr *mr, u32 num_pbes)
 505 {
 506         u32 num_pbls = 0;
 507         u32 idx = 0;
 508         int status = 0;
 509         u32 pbl_size;
 510
 511         do {
 512                 pbl_size = OCRDMA_MIN_HPAGE_SIZE * (1 << idx);
 513                 if (pbl_size > MAX_OCRDMA_PBL_SIZE) {
 514                         status = -EFAULT;
 515                         break;
 516                 }
 517                 num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64)));
 518                 num_pbls = num_pbls / (pbl_size / sizeof(u64));
 519                 idx++;
 520         } while (num_pbls >= mr->hwmr.dev->attr.max_num_mr_pbl);
 521
 522         mr->hwmr.num_pbes = num_pbes;
 523         mr->hwmr.num_pbls = num_pbls;
 524         mr->hwmr.pbl_size = pbl_size;
 525         return status;
 526 }
 527
 528 static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr)
 529 {
 530         int status = 0;
 531         int i;
 532         u32 dma_len = mr->pbl_size;
 533         struct pci_dev *pdev = dev->nic_info.pdev;
 534         void *va;
 535         dma_addr_t pa;
 536
 537         mr->pbl_table = kzalloc(sizeof(struct ocrdma_pbl) *
 538                                 mr->num_pbls, GFP_KERNEL);
 539
 540         if (!mr->pbl_table)
 541                 return -ENOMEM;
 542
 543         for (i = 0; i < mr->num_pbls; i++) {
 544                 va = dma_alloc_coherent(&pdev->dev, dma_len, &pa, GFP_KERNEL);
 545                 if (!va) {
 546                         ocrdma_free_mr_pbl_tbl(dev, mr);
 547                         status = -ENOMEM;
 548                         break;
 549                 }
 550                 memset(va, 0, dma_len);
 551                 mr->pbl_table[i].va = va;
 552                 mr->pbl_table[i].pa = pa;
 553         }
 554         return status;
 555 }
 556
 557 static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
 558                             u32 num_pbes)
 559 {
 560         struct ocrdma_pbe *pbe;
 561         struct ib_umem_chunk *chunk;
 562         struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;
 563         struct ib_umem *umem = mr->umem;
 564         int i, shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
 565
 566         if (!mr->hwmr.num_pbes)
 567                 return;
 568
 569         pbe = (struct ocrdma_pbe *)pbl_tbl->va;
 570         pbe_cnt = 0;
 571
 572         shift = ilog2(umem->page_size);
 573
 574         list_for_each_entry(chunk, &umem->chunk_list, list) {
 575                 /* get all the dma regions from the chunk. */
 576                 for (i = 0; i < chunk->nmap; i++) {
 577                         pages = sg_dma_len(&chunk->page_list[i]) >> shift;
 578                         for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
 579                                 /* store the page address in pbe */
 580                                 pbe->pa_lo =
 581                                     cpu_to_le32(sg_dma_address
 582                                                 (&chunk->page_list[i]) +
 583                                                 (umem->page_size * pg_cnt));
 584                                 pbe->pa_hi =
 585                                     cpu_to_le32(upper_32_bits
 586                                                 ((sg_dma_address
 587                                                   (&chunk->page_list[i]) +
 588                                                   umem->page_size * pg_cnt)));
 589                                 pbe_cnt += 1;
 590                                 total_num_pbes += 1;
 591                                 pbe++;
 592
 593                                 /* if done building pbes, issue the mbx cmd. */
 594                                 if (total_num_pbes == num_pbes)
 595                                         return;
 596
 597                                 /* if the given pbl is full storing the pbes,
 598                                  * move to next pbl.
 599                                  */
 600                                 if (pbe_cnt ==
 601                                         (mr->hwmr.pbl_size / sizeof(u64))) {
 602                                         pbl_tbl++;
 603                                         pbe = (struct ocrdma_pbe *)pbl_tbl->va;
 604                                         pbe_cnt = 0;
 605                                 }
 606                         }
 607                 }
 608         }
 609 }
 610
 611 struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 612                                  u64 usr_addr, int acc, struct ib_udata *udata)
 613 {
 614         int status = -ENOMEM;
 615         struct ocrdma_dev *dev;
 616         struct ocrdma_mr *mr;
 617         struct ocrdma_pd *pd;
 618         u32 num_pbes;
 619
 620         pd = get_ocrdma_pd(ibpd);
 621         dev = pd->dev;
 622
 623         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
 624                 return ERR_PTR(-EINVAL);
 625
 626         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 627         if (!mr)
 628                 return ERR_PTR(status);
 629         mr->hwmr.dev = dev;
 630         mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
 631         if (IS_ERR(mr->umem)) {
 632                 status = -EFAULT;
 633                 goto umem_err;
 634         }
 635         num_pbes = ib_umem_page_count(mr->umem);
 636         status = ocrdma_get_pbl_info(mr, num_pbes);
 637         if (status)
 638                 goto umem_err;
 639
 640         mr->hwmr.pbe_size = mr->umem->page_size;
 641         mr->hwmr.fbo = mr->umem->offset;
 642         mr->hwmr.va = usr_addr;
 643         mr->hwmr.len = len;
 644         mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
 645         mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
 646         mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
 647         mr->hwmr.local_rd = 1;
 648         mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
 649         status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);
 650         if (status)
 651                 goto umem_err;
 652         build_user_pbes(dev, mr, num_pbes);
 653         status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc);
 654         if (status)
 655                 goto mbx_err;
 656         mr->pd = pd;
 657         mr->ibmr.lkey = mr->hwmr.lkey;
 658         if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
 659                 mr->ibmr.rkey = mr->hwmr.lkey;
 660
 661         return &mr->ibmr;
 662
 663 mbx_err:
 664         ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
 665 umem_err:
 666         kfree(mr);
 667         return ERR_PTR(status);
 668 }
 669
 670 int ocrdma_dereg_mr(struct ib_mr *ib_mr)
 671 {
 672         struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr);
 673         struct ocrdma_dev *dev = mr->hwmr.dev;
 674         int status;
 675
 676         status = ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey);
 677
 678         if (mr->hwmr.fr_mr == 0)
 679                 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
 680
 681         /* it could be user registered memory. */
 682         if (mr->umem)
 683                 ib_umem_release(mr->umem);
 684         kfree(mr);
 685         return status;
 686 }
 687
 688 static int ocrdma_copy_cq_uresp(struct ocrdma_cq *cq, struct ib_udata *udata,
 689                                 struct ib_ucontext *ib_ctx)
 690 {
 691         int status;
 692         struct ocrdma_ucontext *uctx;
 693         struct ocrdma_create_cq_uresp uresp;
 694
 695         uresp.cq_id = cq->id;
 696         uresp.page_size = cq->len;
 697         uresp.num_pages = 1;
 698         uresp.max_hw_cqe = cq->max_hw_cqe;
 699         uresp.page_addr[0] = cq->pa;
 700         uresp.db_page_addr = cq->dev->nic_info.unmapped_db;
 701         uresp.db_page_size = cq->dev->nic_info.db_page_size;
 702         uresp.phase_change = cq->phase_change ? 1 : 0;
 703         status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 704         if (status) {
 705                 pr_err("%s(%d) copy error cqid=0x%x.\n",
 706                        __func__, cq->dev->id, cq->id);
 707                 goto err;
 708         }
 709         uctx = get_ocrdma_ucontext(ib_ctx);
 710         status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);
 711         if (status)
 712                 goto err;
 713         status = ocrdma_add_mmap(uctx, uresp.page_addr[0], uresp.page_size);
 714         if (status) {
 715                 ocrdma_del_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);
 716                 goto err;
 717         }
 718         cq->ucontext = uctx;
 719 err:
 720         return status;
 721 }
 722
 723 struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, int entries, int vector,
 724                                struct ib_ucontext *ib_ctx,
 725                                struct ib_udata *udata)
 726 {
 727         struct ocrdma_cq *cq;
 728         struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
 729         int status;
 730         struct ocrdma_create_cq_ureq ureq;
 731
 732         if (udata) {
 733                 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
 734                         return ERR_PTR(-EFAULT);
 735         } else
 736                 ureq.dpp_cq = 0;
 737         cq = kzalloc(sizeof(*cq), GFP_KERNEL);
 738         if (!cq)
 739                 return ERR_PTR(-ENOMEM);
 740
 741         spin_lock_init(&cq->cq_lock);
 742         spin_lock_init(&cq->comp_handler_lock);
 743         INIT_LIST_HEAD(&cq->sq_head);
 744         INIT_LIST_HEAD(&cq->rq_head);
 745         cq->dev = dev;
 746
 747         status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq);
 748         if (status) {
 749                 kfree(cq);
 750                 return ERR_PTR(status);
 751         }
 752         if (ib_ctx) {
 753                 status = ocrdma_copy_cq_uresp(cq, udata, ib_ctx);
 754                 if (status)
 755                         goto ctx_err;
 756         }
 757         cq->phase = OCRDMA_CQE_VALID;
 758         cq->arm_needed = true;
 759         dev->cq_tbl[cq->id] = cq;
 760
 761         return &cq->ibcq;
 762
 763 ctx_err:
 764         ocrdma_mbx_destroy_cq(dev, cq);
 765         kfree(cq);
 766         return ERR_PTR(status);
 767 }
 768
 769 int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt,
 770                      struct ib_udata *udata)
 771 {
 772         int status = 0;
 773         struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
 774
 775         if (new_cnt < 1 || new_cnt > cq->max_hw_cqe) {
 776                 status = -EINVAL;
 777                 return status;
 778         }
 779         ibcq->cqe = new_cnt;
 780         return status;
 781 }
 782
 783 int ocrdma_destroy_cq(struct ib_cq *ibcq)
 784 {
 785         int status;
 786         struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
 787         struct ocrdma_dev *dev = cq->dev;
 788
 789         status = ocrdma_mbx_destroy_cq(dev, cq);
 790
 791         if (cq->ucontext) {
 792                 ocrdma_del_mmap(cq->ucontext, (u64) cq->pa, cq->len);
 793                 ocrdma_del_mmap(cq->ucontext, dev->nic_info.unmapped_db,
 794                                 dev->nic_info.db_page_size);
 795         }
 796         dev->cq_tbl[cq->id] = NULL;
 797
 798         kfree(cq);
 799         return status;
 800 }
 801
 802 static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
 803 {
 804         int status = -EINVAL;
 805
 806         if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) {
 807                 dev->qp_tbl[qp->id] = qp;
 808                 status = 0;
 809         }
 810         return status;
 811 }
 812
 813 static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
 814 {
 815         dev->qp_tbl[qp->id] = NULL;
 816 }
 817
 818 static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev,
 819                                   struct ib_qp_init_attr *attrs)
 820 {
 821         if (attrs->qp_type != IB_QPT_GSI &&
 822             attrs->qp_type != IB_QPT_RC &&
 823             attrs->qp_type != IB_QPT_UD) {
 824                 pr_err("%s(%d) unsupported qp type=0x%x requested\n",
 825                        __func__, dev->id, attrs->qp_type);
 826                 return -EINVAL;
 827         }
 828         if (attrs->cap.max_send_wr > dev->attr.max_wqe) {
 829                 pr_err("%s(%d) unsupported send_wr=0x%x requested\n",
 830                        __func__, dev->id, attrs->cap.max_send_wr);
 831                 pr_err("%s(%d) supported send_wr=0x%x\n",
 832                        __func__, dev->id, dev->attr.max_wqe);
 833                 return -EINVAL;
 834         }
 835         if (!attrs->srq && (attrs->cap.max_recv_wr > dev->attr.max_rqe)) {
 836                 pr_err("%s(%d) unsupported recv_wr=0x%x requested\n",
 837                        __func__, dev->id, attrs->cap.max_recv_wr);
 838                 pr_err("%s(%d) supported recv_wr=0x%x\n",
 839                        __func__, dev->id, dev->attr.max_rqe);
 840                 return -EINVAL;
 841         }
 842         if (attrs->cap.max_inline_data > dev->attr.max_inline_data) {
 843                 pr_err("%s(%d) unsupported inline data size=0x%x requested\n",
 844                        __func__, dev->id, attrs->cap.max_inline_data);
 845                 pr_err("%s(%d) supported inline data size=0x%x\n",
 846                        __func__, dev->id, dev->attr.max_inline_data);
 847                 return -EINVAL;
 848         }
 849         if (attrs->cap.max_send_sge > dev->attr.max_send_sge) {
 850                 pr_err("%s(%d) unsupported send_sge=0x%x requested\n",
 851                        __func__, dev->id, attrs->cap.max_send_sge);
 852                 pr_err("%s(%d) supported send_sge=0x%x\n",
 853                        __func__, dev->id, dev->attr.max_send_sge);
 854                 return -EINVAL;
 855         }
 856         if (attrs->cap.max_recv_sge > dev->attr.max_recv_sge) {
 857                 pr_err("%s(%d) unsupported recv_sge=0x%x requested\n",
 858                        __func__, dev->id, attrs->cap.max_recv_sge);
 859                 pr_err("%s(%d) supported recv_sge=0x%x\n",
 860                        __func__, dev->id, dev->attr.max_recv_sge);
 861                 return -EINVAL;
 862         }
 863         /* unprivileged user space cannot create special QP */
 864         if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
 865                 pr_err
 866                     ("%s(%d) Userspace can't create special QPs of type=0x%x\n",
 867                      __func__, dev->id, attrs->qp_type);
 868                 return -EINVAL;
 869         }
 870         /* allow creating only one GSI type of QP */
 871         if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) {
 872                 pr_err("%s(%d) GSI special QPs already created.\n",
 873                        __func__, dev->id);
 874                 return -EINVAL;
 875         }
 876         /* verify consumer QPs are not trying to use GSI QP's CQ */
 877         if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {
 878                 if ((dev->gsi_sqcq == get_ocrdma_cq(attrs->send_cq)) ||
 879                     (dev->gsi_sqcq == get_ocrdma_cq(attrs->recv_cq)) ||
 880                     (dev->gsi_rqcq == get_ocrdma_cq(attrs->send_cq)) ||
 881                     (dev->gsi_rqcq == get_ocrdma_cq(attrs->recv_cq))) {
 882                         pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n",
 883                                __func__, dev->id);
 884                         return -EINVAL;
 885                 }
 886         }
 887         return 0;
 888 }
 889
 890 static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp,
 891                                 struct ib_udata *udata, int dpp_offset,
 892                                 int dpp_credit_lmt, int srq)
 893 {
 894         int status = 0;
 895         u64 usr_db;
 896         struct ocrdma_create_qp_uresp uresp;
 897         struct ocrdma_dev *dev = qp->dev;
 898         struct ocrdma_pd *pd = qp->pd;
 899
 900         memset(&uresp, 0, sizeof(uresp));
 901         usr_db = dev->nic_info.unmapped_db +
 902                         (pd->id * dev->nic_info.db_page_size);
 903         uresp.qp_id = qp->id;
 904         uresp.sq_dbid = qp->sq.dbid;
 905         uresp.num_sq_pages = 1;
 906         uresp.sq_page_size = qp->sq.len;
 907         uresp.sq_page_addr[0] = qp->sq.pa;
 908         uresp.num_wqe_allocated = qp->sq.max_cnt;
 909         if (!srq) {
 910                 uresp.rq_dbid = qp->rq.dbid;
 911                 uresp.num_rq_pages = 1;
 912                 uresp.rq_page_size = qp->rq.len;
 913                 uresp.rq_page_addr[0] = qp->rq.pa;
 914                 uresp.num_rqe_allocated = qp->rq.max_cnt;
 915         }
 916         uresp.db_page_addr = usr_db;
 917         uresp.db_page_size = dev->nic_info.db_page_size;
 918         if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) {
 919                 uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET;
 920                 uresp.db_rq_offset = ((qp->id & 0xFFFF) < 128) ?
 921                         OCRDMA_DB_GEN2_RQ1_OFFSET : OCRDMA_DB_GEN2_RQ2_OFFSET;
 922                 uresp.db_shift = (qp->id < 128) ? 24 : 16;
 923         } else {
 924                 uresp.db_sq_offset = OCRDMA_DB_SQ_OFFSET;
 925                 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET;
 926                 uresp.db_shift = 16;
 927         }
 928
 929         if (qp->dpp_enabled) {
 930                 uresp.dpp_credit = dpp_credit_lmt;
 931                 uresp.dpp_offset = dpp_offset;
 932         }
 933         status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 934         if (status) {
 935                 pr_err("%s(%d) user copy error.\n", __func__, dev->id);
 936                 goto err;
 937         }
 938         status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0],
 939                                  uresp.sq_page_size);
 940         if (status)
 941                 goto err;
 942
 943         if (!srq) {
 944                 status = ocrdma_add_mmap(pd->uctx, uresp.rq_page_addr[0],
 945                                          uresp.rq_page_size);
 946                 if (status)
 947                         goto rq_map_err;
 948         }
 949         return status;
 950 rq_map_err:
 951         ocrdma_del_mmap(pd->uctx, uresp.sq_page_addr[0], uresp.sq_page_size);
 952 err:
 953         return status;
 954 }
 955
 956 static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp,
 957                              struct ocrdma_pd *pd)
 958 {
 959         if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) {
 960                 qp->sq_db = dev->nic_info.db +
 961                         (pd->id * dev->nic_info.db_page_size) +
 962                         OCRDMA_DB_GEN2_SQ_OFFSET;
 963                 qp->rq_db = dev->nic_info.db +
 964                         (pd->id * dev->nic_info.db_page_size) +
 965                         ((qp->id < 128) ?
 966                         OCRDMA_DB_GEN2_RQ1_OFFSET : OCRDMA_DB_GEN2_RQ2_OFFSET);
 967         } else {
 968                 qp->sq_db = dev->nic_info.db +
 969                         (pd->id * dev->nic_info.db_page_size) +
 970                         OCRDMA_DB_SQ_OFFSET;
 971                 qp->rq_db = dev->nic_info.db +
 972                         (pd->id * dev->nic_info.db_page_size) +
 973                         OCRDMA_DB_RQ_OFFSET;
 974         }
 975 }
 976
 977 static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp)
 978 {
 979         qp->wqe_wr_id_tbl =
 980             kzalloc(sizeof(*(qp->wqe_wr_id_tbl)) * qp->sq.max_cnt,
 981                     GFP_KERNEL);
 982         if (qp->wqe_wr_id_tbl == NULL)
 983                 return -ENOMEM;
 984         qp->rqe_wr_id_tbl =
 985             kzalloc(sizeof(u64) * qp->rq.max_cnt, GFP_KERNEL);
 986         if (qp->rqe_wr_id_tbl == NULL)
 987                 return -ENOMEM;
 988
 989         return 0;
 990 }
 991
 992 static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp,
 993                                       struct ocrdma_pd *pd,
 994                                       struct ib_qp_init_attr *attrs)
 995 {
 996         qp->pd = pd;
 997         spin_lock_init(&qp->q_lock);
 998         INIT_LIST_HEAD(&qp->sq_entry);
 999         INIT_LIST_HEAD(&qp->rq_entry);
1000
1001         qp->qp_type = attrs->qp_type;
1002         qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR;
1003         qp->max_inline_data = attrs->cap.max_inline_data;
1004         qp->sq.max_sges = attrs->cap.max_send_sge;
1005         qp->rq.max_sges = attrs->cap.max_recv_sge;
1006         qp->state = OCRDMA_QPS_RST;
1007 }
1008
1009
1010 static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev,
1011                                    struct ib_qp_init_attr *attrs)
1012 {
1013         if (attrs->qp_type == IB_QPT_GSI) {
1014                 dev->gsi_qp_created = 1;
1015                 dev->gsi_sqcq = get_ocrdma_cq(attrs->send_cq);
1016                 dev->gsi_rqcq = get_ocrdma_cq(attrs->recv_cq);
1017         }
1018 }
1019
1020 struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd,
1021                                struct ib_qp_init_attr *attrs,
1022                                struct ib_udata *udata)
1023 {
1024         int status;
1025         struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
1026         struct ocrdma_qp *qp;
1027         struct ocrdma_dev *dev = pd->dev;
1028         struct ocrdma_create_qp_ureq ureq;
1029         u16 dpp_credit_lmt, dpp_offset;
1030
1031         status = ocrdma_check_qp_params(ibpd, dev, attrs);
1032         if (status)
1033                 goto gen_err;
1034
1035         memset(&ureq, 0, sizeof(ureq));
1036         if (udata) {
1037                 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
1038                         return ERR_PTR(-EFAULT);
1039         }
1040         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1041         if (!qp) {
1042                 status = -ENOMEM;
1043                 goto gen_err;
1044         }
1045         qp->dev = dev;
1046         ocrdma_set_qp_init_params(qp, pd, attrs);
1047
1048         mutex_lock(&dev->dev_lock);
1049         status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq,
1050                                         ureq.dpp_cq_id,
1051                                         &dpp_offset, &dpp_credit_lmt);
1052         if (status)
1053                 goto mbx_err;
1054
1055         /* user space QP's wr_id table are managed in library */
1056         if (udata == NULL) {
1057                 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 |
1058                                   OCRDMA_QP_FAST_REG);
1059                 status = ocrdma_alloc_wr_id_tbl(qp);
1060                 if (status)
1061                         goto map_err;
1062         }
1063
1064         status = ocrdma_add_qpn_map(dev, qp);
1065         if (status)
1066                 goto map_err;
1067         ocrdma_set_qp_db(dev, qp, pd);
1068         if (udata) {
1069                 status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset,
1070                                               dpp_credit_lmt,
1071                                               (attrs->srq != NULL));
1072                 if (status)
1073                         goto cpy_err;
1074         }
1075         ocrdma_store_gsi_qp_cq(dev, attrs);
1076         qp->ibqp.qp_num = qp->id;
1077         mutex_unlock(&dev->dev_lock);
1078         return &qp->ibqp;
1079
1080 cpy_err:
1081         ocrdma_del_qpn_map(dev, qp);
1082 map_err:
1083         ocrdma_mbx_destroy_qp(dev, qp);
1084 mbx_err:
1085         mutex_unlock(&dev->dev_lock);
1086         kfree(qp->wqe_wr_id_tbl);
1087         kfree(qp->rqe_wr_id_tbl);
1088         kfree(qp);
1089         pr_err("%s(%d) error=%d\n", __func__, dev->id, status);
1090 gen_err:
1091         return ERR_PTR(status);
1092 }
1093
1094 int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1095                       int attr_mask)
1096 {
1097         int status = 0;
1098         struct ocrdma_qp *qp;
1099         struct ocrdma_dev *dev;
1100         enum ib_qp_state old_qps;
1101
1102         qp = get_ocrdma_qp(ibqp);
1103         dev = qp->dev;
1104         if (attr_mask & IB_QP_STATE)
1105                 status = ocrdma_qp_state_machine(qp, attr->qp_state, &old_qps);
1106         /* if new and previous states are same hw doesn't need to
1107          * know about it.
1108          */
1109         if (status < 0)
1110                 return status;
1111         status = ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask, old_qps);
1112         return status;
1113 }
1114
1115 int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1116                      int attr_mask, struct ib_udata *udata)
1117 {
1118         unsigned long flags;
1119         int status = -EINVAL;
1120         struct ocrdma_qp *qp;
1121         struct ocrdma_dev *dev;
1122         enum ib_qp_state old_qps, new_qps;
1123
1124         qp = get_ocrdma_qp(ibqp);
1125         dev = qp->dev;
1126
1127         /* syncronize with multiple context trying to change, retrive qps */
1128         mutex_lock(&dev->dev_lock);
1129         /* syncronize with wqe, rqe posting and cqe processing contexts */
1130         spin_lock_irqsave(&qp->q_lock, flags);
1131         old_qps = get_ibqp_state(qp->state);
1132         if (attr_mask & IB_QP_STATE)
1133                 new_qps = attr->qp_state;
1134         else
1135                 new_qps = old_qps;
1136         spin_unlock_irqrestore(&qp->q_lock, flags);
1137
1138         if (!ib_modify_qp_is_ok(old_qps, new_qps, ibqp->qp_type, attr_mask)) {
1139                 pr_err("%s(%d) invalid attribute mask=0x%x specified for\n"
1140                        "qpn=0x%x of type=0x%x old_qps=0x%x, new_qps=0x%x\n",
1141                        __func__, dev->id, attr_mask, qp->id, ibqp->qp_type,
1142                        old_qps, new_qps);
1143                 goto param_err;
1144         }
1145
1146         status = _ocrdma_modify_qp(ibqp, attr, attr_mask);
1147         if (status > 0)
1148                 status = 0;
1149 param_err:
1150         mutex_unlock(&dev->dev_lock);
1151         return status;
1152 }
1153
1154 static enum ib_mtu ocrdma_mtu_int_to_enum(u16 mtu)
1155 {
1156         switch (mtu) {
1157         case 256:
1158                 return IB_MTU_256;
1159         case 512:
1160                 return IB_MTU_512;
1161         case 1024:
1162                 return IB_MTU_1024;
1163         case 2048:
1164                 return IB_MTU_2048;
1165         case 4096:
1166                 return IB_MTU_4096;
1167         default:
1168                 return IB_MTU_1024;
1169         }
1170 }
1171
1172 static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags)
1173 {
1174         int ib_qp_acc_flags = 0;
1175
1176         if (qp_cap_flags & OCRDMA_QP_INB_WR)
1177                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
1178         if (qp_cap_flags & OCRDMA_QP_INB_RD)
1179                 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
1180         return ib_qp_acc_flags;
1181 }
1182
1183 int ocrdma_query_qp(struct ib_qp *ibqp,
1184                     struct ib_qp_attr *qp_attr,
1185                     int attr_mask, struct ib_qp_init_attr *qp_init_attr)
1186 {
1187         int status;
1188         u32 qp_state;
1189         struct ocrdma_qp_params params;
1190         struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
1191         struct ocrdma_dev *dev = qp->dev;
1192
1193         memset(&params, 0, sizeof(params));
1194         mutex_lock(&dev->dev_lock);
1195         status = ocrdma_mbx_query_qp(dev, qp, &params);
1196         mutex_unlock(&dev->dev_lock);
1197         if (status)
1198                 goto mbx_err;
1199         qp_attr->qp_state = get_ibqp_state(IB_QPS_INIT);
1200         qp_attr->cur_qp_state = get_ibqp_state(IB_QPS_INIT);
1201         qp_attr->path_mtu =
1202                 ocrdma_mtu_int_to_enum(params.path_mtu_pkey_indx &
1203                                 OCRDMA_QP_PARAMS_PATH_MTU_MASK) >>
1204                                 OCRDMA_QP_PARAMS_PATH_MTU_SHIFT;
1205         qp_attr->path_mig_state = IB_MIG_MIGRATED;
1206         qp_attr->rq_psn = params.hop_lmt_rq_psn & OCRDMA_QP_PARAMS_RQ_PSN_MASK;
1207         qp_attr->sq_psn = params.tclass_sq_psn & OCRDMA_QP_PARAMS_SQ_PSN_MASK;
1208         qp_attr->dest_qp_num =
1209             params.ack_to_rnr_rtc_dest_qpn & OCRDMA_QP_PARAMS_DEST_QPN_MASK;
1210
1211         qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags);
1212         qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1;
1213         qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1;
1214         qp_attr->cap.max_send_sge = qp->sq.max_sges;
1215         qp_attr->cap.max_recv_sge = qp->rq.max_sges;
1216         qp_attr->cap.max_inline_data = dev->attr.max_inline_data;
1217         qp_init_attr->cap = qp_attr->cap;
1218         memcpy(&qp_attr->ah_attr.grh.dgid, &params.dgid[0],
1219                sizeof(params.dgid));
1220         qp_attr->ah_attr.grh.flow_label = params.rnt_rc_sl_fl &
1221             OCRDMA_QP_PARAMS_FLOW_LABEL_MASK;
1222         qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
1223         qp_attr->ah_attr.grh.hop_limit = (params.hop_lmt_rq_psn &
1224                                           OCRDMA_QP_PARAMS_HOP_LMT_MASK) >>
1225                                                 OCRDMA_QP_PARAMS_HOP_LMT_SHIFT;
1226         qp_attr->ah_attr.grh.traffic_class = (params.tclass_sq_psn &
1227                                               OCRDMA_QP_PARAMS_SQ_PSN_MASK) >>
1228                                                 OCRDMA_QP_PARAMS_TCLASS_SHIFT;
1229
1230         qp_attr->ah_attr.ah_flags = IB_AH_GRH;
1231         qp_attr->ah_attr.port_num = 1;
1232         qp_attr->ah_attr.sl = (params.rnt_rc_sl_fl &
1233                                OCRDMA_QP_PARAMS_SL_MASK) >>
1234                                 OCRDMA_QP_PARAMS_SL_SHIFT;
1235         qp_attr->timeout = (params.ack_to_rnr_rtc_dest_qpn &
1236                             OCRDMA_QP_PARAMS_ACK_TIMEOUT_MASK) >>
1237                                 OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT;
1238         qp_attr->rnr_retry = (params.ack_to_rnr_rtc_dest_qpn &
1239                               OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK) >>
1240                                 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT;
1241         qp_attr->retry_cnt =
1242             (params.rnt_rc_sl_fl & OCRDMA_QP_PARAMS_RETRY_CNT_MASK) >>
1243                 OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT;
1244         qp_attr->min_rnr_timer = 0;
1245         qp_attr->pkey_index = 0;
1246         qp_attr->port_num = 1;
1247         qp_attr->ah_attr.src_path_bits = 0;
1248         qp_attr->ah_attr.static_rate = 0;
1249         qp_attr->alt_pkey_index = 0;
1250         qp_attr->alt_port_num = 0;
1251         qp_attr->alt_timeout = 0;
1252         memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
1253         qp_state = (params.max_sge_recv_flags & OCRDMA_QP_PARAMS_STATE_MASK) >>
1254                     OCRDMA_QP_PARAMS_STATE_SHIFT;
1255         qp_attr->sq_draining = (qp_state == OCRDMA_QPS_SQ_DRAINING) ? 1 : 0;
1256         qp_attr->max_dest_rd_atomic =
1257             params.max_ord_ird >> OCRDMA_QP_PARAMS_MAX_ORD_SHIFT;
1258         qp_attr->max_rd_atomic =
1259             params.max_ord_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK;
1260         qp_attr->en_sqd_async_notify = (params.max_sge_recv_flags &
1261                                 OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC) ? 1 : 0;
1262 mbx_err:
1263         return status;
1264 }
1265
1266 static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, int idx)
1267 {
1268         int i = idx / 32;
1269         unsigned int mask = (1 << (idx % 32));
1270
1271         if (srq->idx_bit_fields[i] & mask)
1272                 srq->idx_bit_fields[i] &= ~mask;
1273         else
1274                 srq->idx_bit_fields[i] |= mask;
1275 }
1276
1277 static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q)
1278 {
1279         int free_cnt;
1280         if (q->head >= q->tail)
1281                 free_cnt = (q->max_cnt - q->head) + q->tail;
1282         else
1283                 free_cnt = q->tail - q->head;
1284         return free_cnt;
1285 }
1286
1287 static int is_hw_sq_empty(struct ocrdma_qp *qp)
1288 {
1289         return (qp->sq.tail == qp->sq.head &&
1290                 ocrdma_hwq_free_cnt(&qp->sq) ? 1 : 0);
1291 }
1292
1293 static int is_hw_rq_empty(struct ocrdma_qp *qp)
1294 {
1295         return (qp->rq.tail == qp->rq.head) ? 1 : 0;
1296 }
1297
1298 static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q)
1299 {
1300         return q->va + (q->head * q->entry_size);
1301 }
1302
1303 static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q,
1304                                       u32 idx)
1305 {
1306         return q->va + (idx * q->entry_size);
1307 }
1308
1309 static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q)
1310 {
1311         q->head = (q->head + 1) & q->max_wqe_idx;
1312 }
1313
1314 static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q)
1315 {
1316         q->tail = (q->tail + 1) & q->max_wqe_idx;
1317 }
1318
1319 /* discard the cqe for a given QP */
1320 static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)
1321 {
1322         unsigned long cq_flags;
1323         unsigned long flags;
1324         int discard_cnt = 0;
1325         u32 cur_getp, stop_getp;
1326         struct ocrdma_cqe *cqe;
1327         u32 qpn = 0;
1328
1329         spin_lock_irqsave(&cq->cq_lock, cq_flags);
1330
1331         /* traverse through the CQEs in the hw CQ,
1332          * find the matching CQE for a given qp,
1333          * mark the matching one discarded by clearing qpn.
1334          * ring the doorbell in the poll_cq() as
1335          * we don't complete out of order cqe.
1336          */
1337
1338         cur_getp = cq->getp;
1339         /* find upto when do we reap the cq. */
1340         stop_getp = cur_getp;
1341         do {
1342                 if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp)))
1343                         break;
1344
1345                 cqe = cq->va + cur_getp;
1346                 /* if (a) done reaping whole hw cq, or
1347                  *    (b) qp_xq becomes empty.
1348                  * then exit
1349                  */
1350                 qpn = cqe->cmn.qpn & OCRDMA_CQE_QPN_MASK;
1351                 /* if previously discarded cqe found, skip that too. */
1352                 /* check for matching qp */
1353                 if (qpn == 0 || qpn != qp->id)
1354                         goto skip_cqe;
1355
1356                 /* mark cqe discarded so that it is not picked up later
1357                  * in the poll_cq().
1358                  */
1359                 discard_cnt += 1;
1360                 cqe->cmn.qpn = 0;
1361                 if (is_cqe_for_sq(cqe))
1362                         ocrdma_hwq_inc_tail(&qp->sq);
1363                 else {
1364                         if (qp->srq) {
1365                                 spin_lock_irqsave(&qp->srq->q_lock, flags);
1366                                 ocrdma_hwq_inc_tail(&qp->srq->rq);
1367                                 ocrdma_srq_toggle_bit(qp->srq, cur_getp);
1368                                 spin_unlock_irqrestore(&qp->srq->q_lock, flags);
1369
1370                         } else
1371                                 ocrdma_hwq_inc_tail(&qp->rq);
1372                 }
1373 skip_cqe:
1374                 cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
1375         } while (cur_getp != stop_getp);
1376         spin_unlock_irqrestore(&cq->cq_lock, cq_flags);
1377 }
1378
1379 static void ocrdma_del_flush_qp(struct ocrdma_qp *qp)
1380 {
1381         int found = false;
1382         unsigned long flags;
1383         struct ocrdma_dev *dev = qp->dev;
1384         /* sync with any active CQ poll */
1385
1386         spin_lock_irqsave(&dev->flush_q_lock, flags);
1387         found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp);
1388         if (found)
1389                 list_del(&qp->sq_entry);
1390         if (!qp->srq) {
1391                 found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp);
1392                 if (found)
1393                         list_del(&qp->rq_entry);
1394         }
1395         spin_unlock_irqrestore(&dev->flush_q_lock, flags);
1396 }
1397
1398 int ocrdma_destroy_qp(struct ib_qp *ibqp)
1399 {
1400         int status;
1401         struct ocrdma_pd *pd;
1402         struct ocrdma_qp *qp;
1403         struct ocrdma_dev *dev;
1404         struct ib_qp_attr attrs;
1405         int attr_mask = IB_QP_STATE;
1406         unsigned long flags;
1407
1408         qp = get_ocrdma_qp(ibqp);
1409         dev = qp->dev;
1410
1411         attrs.qp_state = IB_QPS_ERR;
1412         pd = qp->pd;
1413
1414         /* change the QP state to ERROR */
1415         _ocrdma_modify_qp(ibqp, &attrs, attr_mask);
1416
1417         /* ensure that CQEs for newly created QP (whose id may be same with
1418          * one which just getting destroyed are same), dont get
1419          * discarded until the old CQEs are discarded.
1420          */
1421         mutex_lock(&dev->dev_lock);
1422         status = ocrdma_mbx_destroy_qp(dev, qp);
1423
1424         /*
1425          * acquire CQ lock while destroy is in progress, in order to
1426          * protect against proessing in-flight CQEs for this QP.
1427          */
1428         spin_lock_irqsave(&qp->sq_cq->cq_lock, flags);
1429         if (qp->rq_cq && (qp->rq_cq != qp->sq_cq))
1430                 spin_lock(&qp->rq_cq->cq_lock);
1431
1432         ocrdma_del_qpn_map(dev, qp);
1433
1434         if (qp->rq_cq && (qp->rq_cq != qp->sq_cq))
1435                 spin_unlock(&qp->rq_cq->cq_lock);
1436         spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags);
1437
1438         if (!pd->uctx) {
1439                 ocrdma_discard_cqes(qp, qp->sq_cq);
1440                 ocrdma_discard_cqes(qp, qp->rq_cq);
1441         }
1442         mutex_unlock(&dev->dev_lock);
1443
1444         if (pd->uctx) {
1445                 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa, qp->sq.len);
1446                 if (!qp->srq)
1447                         ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa, qp->rq.len);
1448         }
1449
1450         ocrdma_del_flush_qp(qp);
1451
1452         kfree(qp->wqe_wr_id_tbl);
1453         kfree(qp->rqe_wr_id_tbl);
1454         kfree(qp);
1455         return status;
1456 }
1457
1458 static int ocrdma_copy_srq_uresp(struct ocrdma_srq *srq, struct ib_udata *udata)
1459 {
1460         int status;
1461         struct ocrdma_create_srq_uresp uresp;
1462
1463         uresp.rq_dbid = srq->rq.dbid;
1464         uresp.num_rq_pages = 1;
1465         uresp.rq_page_addr[0] = srq->rq.pa;
1466         uresp.rq_page_size = srq->rq.len;
1467         uresp.db_page_addr = srq->dev->nic_info.unmapped_db +
1468             (srq->pd->id * srq->dev->nic_info.db_page_size);
1469         uresp.db_page_size = srq->dev->nic_info.db_page_size;
1470         uresp.num_rqe_allocated = srq->rq.max_cnt;
1471         if (srq->dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) {
1472                 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ1_OFFSET;
1473                 uresp.db_shift = 24;
1474         } else {
1475                 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET;
1476                 uresp.db_shift = 16;
1477         }
1478
1479         status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1480         if (status)
1481                 return status;
1482         status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0],
1483                                  uresp.rq_page_size);
1484         if (status)
1485                 return status;
1486         return status;
1487 }
1488
1489 struct ib_srq *ocrdma_create_srq(struct ib_pd *ibpd,
1490                                  struct ib_srq_init_attr *init_attr,
1491                                  struct ib_udata *udata)
1492 {
1493         int status = -ENOMEM;
1494         struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
1495         struct ocrdma_dev *dev = pd->dev;
1496         struct ocrdma_srq *srq;
1497
1498         if (init_attr->attr.max_sge > dev->attr.max_recv_sge)
1499                 return ERR_PTR(-EINVAL);
1500         if (init_attr->attr.max_wr > dev->attr.max_rqe)
1501                 return ERR_PTR(-EINVAL);
1502
1503         srq = kzalloc(sizeof(*srq), GFP_KERNEL);
1504         if (!srq)
1505                 return ERR_PTR(status);
1506
1507         spin_lock_init(&srq->q_lock);
1508         srq->dev = dev;
1509         srq->pd = pd;
1510         srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size);
1511         status = ocrdma_mbx_create_srq(srq, init_attr, pd);
1512         if (status)
1513                 goto err;
1514
1515         if (udata == NULL) {
1516                 srq->rqe_wr_id_tbl = kzalloc(sizeof(u64) * srq->rq.max_cnt,
1517                             GFP_KERNEL);
1518                 if (srq->rqe_wr_id_tbl == NULL)
1519                         goto arm_err;
1520
1521                 srq->bit_fields_len = (srq->rq.max_cnt / 32) +
1522                     (srq->rq.max_cnt % 32 ? 1 : 0);
1523                 srq->idx_bit_fields =
1524                     kmalloc(srq->bit_fields_len * sizeof(u32), GFP_KERNEL);
1525                 if (srq->idx_bit_fields == NULL)
1526                         goto arm_err;
1527                 memset(srq->idx_bit_fields, 0xff,
1528                        srq->bit_fields_len * sizeof(u32));
1529         }
1530
1531         if (init_attr->attr.srq_limit) {
1532                 status = ocrdma_mbx_modify_srq(srq, &init_attr->attr);
1533                 if (status)
1534                         goto arm_err;
1535         }
1536
1537         if (udata) {
1538                 status = ocrdma_copy_srq_uresp(srq, udata);
1539                 if (status)
1540                         goto arm_err;
1541         }
1542
1543         return &srq->ibsrq;
1544
1545 arm_err:
1546         ocrdma_mbx_destroy_srq(dev, srq);
1547 err:
1548         kfree(srq->rqe_wr_id_tbl);
1549         kfree(srq->idx_bit_fields);
1550         kfree(srq);
1551         return ERR_PTR(status);
1552 }
1553
1554 int ocrdma_modify_srq(struct ib_srq *ibsrq,
1555                       struct ib_srq_attr *srq_attr,
1556                       enum ib_srq_attr_mask srq_attr_mask,
1557                       struct ib_udata *udata)
1558 {
1559         int status = 0;
1560         struct ocrdma_srq *srq;
1561
1562         srq = get_ocrdma_srq(ibsrq);
1563         if (srq_attr_mask & IB_SRQ_MAX_WR)
1564                 status = -EINVAL;
1565         else
1566                 status = ocrdma_mbx_modify_srq(srq, srq_attr);
1567         return status;
1568 }
1569
1570 int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
1571 {
1572         int status;
1573         struct ocrdma_srq *srq;
1574
1575         srq = get_ocrdma_srq(ibsrq);
1576         status = ocrdma_mbx_query_srq(srq, srq_attr);
1577         return status;
1578 }
1579
1580 int ocrdma_destroy_srq(struct ib_srq *ibsrq)
1581 {
1582         int status;
1583         struct ocrdma_srq *srq;
1584         struct ocrdma_dev *dev;
1585
1586         srq = get_ocrdma_srq(ibsrq);
1587         dev = srq->dev;
1588
1589         status = ocrdma_mbx_destroy_srq(dev, srq);
1590
1591         if (srq->pd->uctx)
1592                 ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa, srq->rq.len);
1593
1594         kfree(srq->idx_bit_fields);
1595         kfree(srq->rqe_wr_id_tbl);
1596         kfree(srq);
1597         return status;
1598 }
1599
1600 /* unprivileged verbs and their support functions. */
1601 static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp,
1602                                 struct ocrdma_hdr_wqe *hdr,
1603                                 struct ib_send_wr *wr)
1604 {
1605         struct ocrdma_ewqe_ud_hdr *ud_hdr =
1606                 (struct ocrdma_ewqe_ud_hdr *)(hdr + 1);
1607         struct ocrdma_ah *ah = get_ocrdma_ah(wr->wr.ud.ah);
1608
1609         ud_hdr->rsvd_dest_qpn = wr->wr.ud.remote_qpn;
1610         if (qp->qp_type == IB_QPT_GSI)
1611                 ud_hdr->qkey = qp->qkey;
1612         else
1613                 ud_hdr->qkey = wr->wr.ud.remote_qkey;
1614         ud_hdr->rsvd_ahid = ah->id;
1615 }
1616
1617 static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr,
1618                               struct ocrdma_sge *sge, int num_sge,
1619                               struct ib_sge *sg_list)
1620 {
1621         int i;
1622
1623         for (i = 0; i < num_sge; i++) {
1624                 sge[i].lrkey = sg_list[i].lkey;
1625                 sge[i].addr_lo = sg_list[i].addr;
1626                 sge[i].addr_hi = upper_32_bits(sg_list[i].addr);
1627                 sge[i].len = sg_list[i].length;
1628                 hdr->total_len += sg_list[i].length;
1629         }
1630         if (num_sge == 0)
1631                 memset(sge, 0, sizeof(*sge));
1632 }
1633
1634 static int ocrdma_build_inline_sges(struct ocrdma_qp *qp,
1635                                     struct ocrdma_hdr_wqe *hdr,
1636                                     struct ocrdma_sge *sge,
1637                                     struct ib_send_wr *wr, u32 wqe_size)
1638 {
1639         if (wr->send_flags & IB_SEND_INLINE) {
1640                 if (wr->sg_list[0].length > qp->max_inline_data) {
1641                         pr_err("%s() supported_len=0x%x,\n"
1642                                " unspported len req=0x%x\n", __func__,
1643                                qp->max_inline_data, wr->sg_list[0].length);
1644                         return -EINVAL;
1645                 }
1646                 memcpy(sge,
1647                        (void *)(unsigned long)wr->sg_list[0].addr,
1648                        wr->sg_list[0].length);
1649                 hdr->total_len = wr->sg_list[0].length;
1650                 wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES);
1651                 hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT);
1652         } else {
1653                 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);
1654                 if (wr->num_sge)
1655                         wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge));
1656                 else
1657                         wqe_size += sizeof(struct ocrdma_sge);
1658                 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
1659         }
1660         hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
1661         return 0;
1662 }
1663
1664 static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1665                              struct ib_send_wr *wr)
1666 {
1667         int status;
1668         struct ocrdma_sge *sge;
1669         u32 wqe_size = sizeof(*hdr);
1670
1671         if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
1672                 ocrdma_build_ud_hdr(qp, hdr, wr);
1673                 sge = (struct ocrdma_sge *)(hdr + 2);
1674                 wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr);
1675         } else
1676                 sge = (struct ocrdma_sge *)(hdr + 1);
1677
1678         status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1679         return status;
1680 }
1681
1682 static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1683                               struct ib_send_wr *wr)
1684 {
1685         int status;
1686         struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
1687         struct ocrdma_sge *sge = ext_rw + 1;
1688         u32 wqe_size = sizeof(*hdr) + sizeof(*ext_rw);
1689
1690         status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1691         if (status)
1692                 return status;
1693         ext_rw->addr_lo = wr->wr.rdma.remote_addr;
1694         ext_rw->addr_hi = upper_32_bits(wr->wr.rdma.remote_addr);
1695         ext_rw->lrkey = wr->wr.rdma.rkey;
1696         ext_rw->len = hdr->total_len;
1697         return 0;
1698 }
1699
1700 static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1701                               struct ib_send_wr *wr)
1702 {
1703         struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
1704         struct ocrdma_sge *sge = ext_rw + 1;
1705         u32 wqe_size = ((wr->num_sge + 1) * sizeof(struct ocrdma_sge)) +
1706             sizeof(struct ocrdma_hdr_wqe);
1707
1708         ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);
1709         hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
1710         hdr->cw |= (OCRDMA_READ << OCRDMA_WQE_OPCODE_SHIFT);
1711         hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
1712
1713         ext_rw->addr_lo = wr->wr.rdma.remote_addr;
1714         ext_rw->addr_hi = upper_32_bits(wr->wr.rdma.remote_addr);
1715         ext_rw->lrkey = wr->wr.rdma.rkey;
1716         ext_rw->len = hdr->total_len;
1717 }
1718
1719 static void ocrdma_ring_sq_db(struct ocrdma_qp *qp)
1720 {
1721         u32 val = qp->sq.dbid | (1 << 16);
1722
1723         iowrite32(val, qp->sq_db);
1724 }
1725
1726 int ocrdma_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
1727                      struct ib_send_wr **bad_wr)
1728 {
1729         int status = 0;
1730         struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
1731         struct ocrdma_hdr_wqe *hdr;
1732         unsigned long flags;
1733
1734         spin_lock_irqsave(&qp->q_lock, flags);
1735         if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) {
1736                 spin_unlock_irqrestore(&qp->q_lock, flags);
1737                 *bad_wr = wr;
1738                 return -EINVAL;
1739         }
1740
1741         while (wr) {
1742                 if (ocrdma_hwq_free_cnt(&qp->sq) == 0 ||
1743                     wr->num_sge > qp->sq.max_sges) {
1744                         *bad_wr = wr;
1745                         status = -ENOMEM;
1746                         break;
1747                 }
1748                 hdr = ocrdma_hwq_head(&qp->sq);
1749                 hdr->cw = 0;
1750                 if (wr->send_flags & IB_SEND_SIGNALED)
1751                         hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
1752                 if (wr->send_flags & IB_SEND_FENCE)
1753                         hdr->cw |=
1754                             (OCRDMA_FLAG_FENCE_L << OCRDMA_WQE_FLAGS_SHIFT);
1755                 if (wr->send_flags & IB_SEND_SOLICITED)
1756                         hdr->cw |=
1757                             (OCRDMA_FLAG_SOLICIT << OCRDMA_WQE_FLAGS_SHIFT);
1758                 hdr->total_len = 0;
1759                 switch (wr->opcode) {
1760                 case IB_WR_SEND_WITH_IMM:
1761                         hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
1762                         hdr->immdt = ntohl(wr->ex.imm_data);
1763                 case IB_WR_SEND:
1764                         hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);
1765                         ocrdma_build_send(qp, hdr, wr);
1766                         break;
1767                 case IB_WR_SEND_WITH_INV:
1768                         hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT);
1769                         hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);
1770                         hdr->lkey = wr->ex.invalidate_rkey;
1771                         status = ocrdma_build_send(qp, hdr, wr);
1772                         break;
1773                 case IB_WR_RDMA_WRITE_WITH_IMM:
1774                         hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
1775                         hdr->immdt = ntohl(wr->ex.imm_data);
1776                 case IB_WR_RDMA_WRITE:
1777                         hdr->cw |= (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT);
1778                         status = ocrdma_build_write(qp, hdr, wr);
1779                         break;
1780                 case IB_WR_RDMA_READ_WITH_INV:
1781                         hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT);
1782                 case IB_WR_RDMA_READ:
1783                         ocrdma_build_read(qp, hdr, wr);
1784                         break;
1785                 case IB_WR_LOCAL_INV:
1786                         hdr->cw |=
1787                             (OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT);
1788                         hdr->cw |= (sizeof(struct ocrdma_hdr_wqe) /
1789                                 OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT;
1790                         hdr->lkey = wr->ex.invalidate_rkey;
1791                         break;
1792                 default:
1793                         status = -EINVAL;
1794                         break;
1795                 }
1796                 if (status) {
1797                         *bad_wr = wr;
1798                         break;
1799                 }
1800                 if (wr->send_flags & IB_SEND_SIGNALED)
1801                         qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1;
1802                 else
1803                         qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0;
1804                 qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id;
1805                 ocrdma_cpu_to_le32(hdr, ((hdr->cw >> OCRDMA_WQE_SIZE_SHIFT) &
1806                                    OCRDMA_WQE_SIZE_MASK) * OCRDMA_WQE_STRIDE);
1807                 /* make sure wqe is written before adapter can access it */
1808                 wmb();
1809                 /* inform hw to start processing it */
1810                 ocrdma_ring_sq_db(qp);
1811
1812                 /* update pointer, counter for next wr */
1813                 ocrdma_hwq_inc_head(&qp->sq);
1814                 wr = wr->next;
1815         }
1816         spin_unlock_irqrestore(&qp->q_lock, flags);
1817         return status;
1818 }
1819
1820 static void ocrdma_ring_rq_db(struct ocrdma_qp *qp)
1821 {
1822         u32 val = qp->rq.dbid | (1 << ocrdma_get_num_posted_shift(qp));
1823
1824         iowrite32(val, qp->rq_db);
1825 }
1826
1827 static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe, struct ib_recv_wr *wr,
1828                              u16 tag)
1829 {
1830         u32 wqe_size = 0;
1831         struct ocrdma_sge *sge;
1832         if (wr->num_sge)
1833                 wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe);
1834         else
1835                 wqe_size = sizeof(*sge) + sizeof(*rqe);
1836
1837         rqe->cw = ((wqe_size / OCRDMA_WQE_STRIDE) <<
1838                                 OCRDMA_WQE_SIZE_SHIFT);
1839         rqe->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
1840         rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
1841         rqe->total_len = 0;
1842         rqe->rsvd_tag = tag;
1843         sge = (struct ocrdma_sge *)(rqe + 1);
1844         ocrdma_build_sges(rqe, sge, wr->num_sge, wr->sg_list);
1845         ocrdma_cpu_to_le32(rqe, wqe_size);
1846 }
1847
1848 int ocrdma_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
1849                      struct ib_recv_wr **bad_wr)
1850 {
1851         int status = 0;
1852         unsigned long flags;
1853         struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
1854         struct ocrdma_hdr_wqe *rqe;
1855
1856         spin_lock_irqsave(&qp->q_lock, flags);
1857         if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) {
1858                 spin_unlock_irqrestore(&qp->q_lock, flags);
1859                 *bad_wr = wr;
1860                 return -EINVAL;
1861         }
1862         while (wr) {
1863                 if (ocrdma_hwq_free_cnt(&qp->rq) == 0 ||
1864                     wr->num_sge > qp->rq.max_sges) {
1865                         *bad_wr = wr;
1866                         status = -ENOMEM;
1867                         break;
1868                 }
1869                 rqe = ocrdma_hwq_head(&qp->rq);
1870                 ocrdma_build_rqe(rqe, wr, 0);
1871
1872                 qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id;
1873                 /* make sure rqe is written before adapter can access it */
1874                 wmb();
1875
1876                 /* inform hw to start processing it */
1877                 ocrdma_ring_rq_db(qp);
1878
1879                 /* update pointer, counter for next wr */
1880                 ocrdma_hwq_inc_head(&qp->rq);
1881                 wr = wr->next;
1882         }
1883         spin_unlock_irqrestore(&qp->q_lock, flags);
1884         return status;
1885 }
1886
1887 /* cqe for srq's rqe can potentially arrive out of order.
1888  * index gives the entry in the shadow table where to store
1889  * the wr_id. tag/index is returned in cqe to reference back
1890  * for a given rqe.
1891  */
1892 static int ocrdma_srq_get_idx(struct ocrdma_srq *srq)
1893 {
1894         int row = 0;
1895         int indx = 0;
1896
1897         for (row = 0; row < srq->bit_fields_len; row++) {
1898                 if (srq->idx_bit_fields[row]) {
1899                         indx = ffs(srq->idx_bit_fields[row]);
1900                         indx = (row * 32) + (indx - 1);
1901                         if (indx >= srq->rq.max_cnt)
1902                                 BUG();
1903                         ocrdma_srq_toggle_bit(srq, indx);
1904                         break;
1905                 }
1906         }
1907
1908         if (row == srq->bit_fields_len)
1909                 BUG();
1910         return indx;
1911 }
1912
1913 static void ocrdma_ring_srq_db(struct ocrdma_srq *srq)
1914 {
1915         u32 val = srq->rq.dbid | (1 << 16);
1916
1917         iowrite32(val, srq->db + OCRDMA_DB_GEN2_SRQ_OFFSET);
1918 }
1919
1920 int ocrdma_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
1921                          struct ib_recv_wr **bad_wr)
1922 {
1923         int status = 0;
1924         unsigned long flags;
1925         struct ocrdma_srq *srq;
1926         struct ocrdma_hdr_wqe *rqe;
1927         u16 tag;
1928
1929         srq = get_ocrdma_srq(ibsrq);
1930
1931         spin_lock_irqsave(&srq->q_lock, flags);
1932         while (wr) {
1933                 if (ocrdma_hwq_free_cnt(&srq->rq) == 0 ||
1934                     wr->num_sge > srq->rq.max_sges) {
1935                         status = -ENOMEM;
1936                         *bad_wr = wr;
1937                         break;
1938                 }
1939                 tag = ocrdma_srq_get_idx(srq);
1940                 rqe = ocrdma_hwq_head(&srq->rq);
1941                 ocrdma_build_rqe(rqe, wr, tag);
1942
1943                 srq->rqe_wr_id_tbl[tag] = wr->wr_id;
1944                 /* make sure rqe is written before adapter can perform DMA */
1945                 wmb();
1946                 /* inform hw to start processing it */
1947                 ocrdma_ring_srq_db(srq);
1948                 /* update pointer, counter for next wr */
1949                 ocrdma_hwq_inc_head(&srq->rq);
1950                 wr = wr->next;
1951         }
1952         spin_unlock_irqrestore(&srq->q_lock, flags);
1953         return status;
1954 }
1955
1956 static enum ib_wc_status ocrdma_to_ibwc_err(u16 status)
1957 {
1958         enum ib_wc_status ibwc_status = IB_WC_GENERAL_ERR;
1959
1960         switch (status) {
1961         case OCRDMA_CQE_GENERAL_ERR:
1962                 ibwc_status = IB_WC_GENERAL_ERR;
1963                 break;
1964         case OCRDMA_CQE_LOC_LEN_ERR:
1965                 ibwc_status = IB_WC_LOC_LEN_ERR;
1966                 break;
1967         case OCRDMA_CQE_LOC_QP_OP_ERR:
1968                 ibwc_status = IB_WC_LOC_QP_OP_ERR;
1969                 break;
1970         case OCRDMA_CQE_LOC_EEC_OP_ERR:
1971                 ibwc_status = IB_WC_LOC_EEC_OP_ERR;
1972                 break;
1973         case OCRDMA_CQE_LOC_PROT_ERR:
1974                 ibwc_status = IB_WC_LOC_PROT_ERR;
1975                 break;
1976         case OCRDMA_CQE_WR_FLUSH_ERR:
1977                 ibwc_status = IB_WC_WR_FLUSH_ERR;
1978                 break;
1979         case OCRDMA_CQE_MW_BIND_ERR:
1980                 ibwc_status = IB_WC_MW_BIND_ERR;
1981                 break;
1982         case OCRDMA_CQE_BAD_RESP_ERR:
1983                 ibwc_status = IB_WC_BAD_RESP_ERR;
1984                 break;
1985         case OCRDMA_CQE_LOC_ACCESS_ERR:
1986                 ibwc_status = IB_WC_LOC_ACCESS_ERR;
1987                 break;
1988         case OCRDMA_CQE_REM_INV_REQ_ERR:
1989                 ibwc_status = IB_WC_REM_INV_REQ_ERR;
1990                 break;
1991         case OCRDMA_CQE_REM_ACCESS_ERR:
1992                 ibwc_status = IB_WC_REM_ACCESS_ERR;
1993                 break;
1994         case OCRDMA_CQE_REM_OP_ERR:
1995                 ibwc_status = IB_WC_REM_OP_ERR;
1996                 break;
1997         case OCRDMA_CQE_RETRY_EXC_ERR:
1998                 ibwc_status = IB_WC_RETRY_EXC_ERR;
1999                 break;
2000         case OCRDMA_CQE_RNR_RETRY_EXC_ERR:
2001                 ibwc_status = IB_WC_RNR_RETRY_EXC_ERR;
2002                 break;
2003         case OCRDMA_CQE_LOC_RDD_VIOL_ERR:
2004                 ibwc_status = IB_WC_LOC_RDD_VIOL_ERR;
2005                 break;
2006         case OCRDMA_CQE_REM_INV_RD_REQ_ERR:
2007                 ibwc_status = IB_WC_REM_INV_RD_REQ_ERR;
2008                 break;
2009         case OCRDMA_CQE_REM_ABORT_ERR:
2010                 ibwc_status = IB_WC_REM_ABORT_ERR;
2011                 break;
2012         case OCRDMA_CQE_INV_EECN_ERR:
2013                 ibwc_status = IB_WC_INV_EECN_ERR;
2014                 break;
2015         case OCRDMA_CQE_INV_EEC_STATE_ERR:
2016                 ibwc_status = IB_WC_INV_EEC_STATE_ERR;
2017                 break;
2018         case OCRDMA_CQE_FATAL_ERR:
2019                 ibwc_status = IB_WC_FATAL_ERR;
2020                 break;
2021         case OCRDMA_CQE_RESP_TIMEOUT_ERR:
2022                 ibwc_status = IB_WC_RESP_TIMEOUT_ERR;
2023                 break;
2024         default:
2025                 ibwc_status = IB_WC_GENERAL_ERR;
2026                 break;
2027         };
2028         return ibwc_status;
2029 }
2030
2031 static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc,
2032                       u32 wqe_idx)
2033 {
2034         struct ocrdma_hdr_wqe *hdr;
2035         struct ocrdma_sge *rw;
2036         int opcode;
2037
2038         hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx);
2039
2040         ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid;
2041         /* Undo the hdr->cw swap */
2042         opcode = le32_to_cpu(hdr->cw) & OCRDMA_WQE_OPCODE_MASK;
2043         switch (opcode) {
2044         case OCRDMA_WRITE:
2045                 ibwc->opcode = IB_WC_RDMA_WRITE;
2046                 break;
2047         case OCRDMA_READ:
2048                 rw = (struct ocrdma_sge *)(hdr + 1);
2049                 ibwc->opcode = IB_WC_RDMA_READ;
2050                 ibwc->byte_len = rw->len;
2051                 break;
2052         case OCRDMA_SEND:
2053                 ibwc->opcode = IB_WC_SEND;
2054                 break;
2055         case OCRDMA_LKEY_INV:
2056                 ibwc->opcode = IB_WC_LOCAL_INV;
2057                 break;
2058         default:
2059                 ibwc->status = IB_WC_GENERAL_ERR;
2060                 pr_err("%s() invalid opcode received = 0x%x\n",
2061                        __func__, hdr->cw & OCRDMA_WQE_OPCODE_MASK);
2062                 break;
2063         };
2064 }
2065
2066 static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp,
2067                                                 struct ocrdma_cqe *cqe)
2068 {
2069         if (is_cqe_for_sq(cqe)) {
2070                 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2071                                 cqe->flags_status_srcqpn) &
2072                                         ~OCRDMA_CQE_STATUS_MASK);
2073                 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2074                                 cqe->flags_status_srcqpn) |
2075                                 (OCRDMA_CQE_WR_FLUSH_ERR <<
2076                                         OCRDMA_CQE_STATUS_SHIFT));
2077         } else {
2078                 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
2079                         cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2080                                         cqe->flags_status_srcqpn) &
2081                                                 ~OCRDMA_CQE_UD_STATUS_MASK);
2082                         cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2083                                         cqe->flags_status_srcqpn) |
2084                                         (OCRDMA_CQE_WR_FLUSH_ERR <<
2085                                                 OCRDMA_CQE_UD_STATUS_SHIFT));
2086                 } else {
2087                         cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2088                                         cqe->flags_status_srcqpn) &
2089                                                 ~OCRDMA_CQE_STATUS_MASK);
2090                         cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2091                                         cqe->flags_status_srcqpn) |
2092                                         (OCRDMA_CQE_WR_FLUSH_ERR <<
2093                                                 OCRDMA_CQE_STATUS_SHIFT));
2094                 }
2095         }
2096 }
2097
2098 static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2099                                   struct ocrdma_qp *qp, int status)
2100 {
2101         bool expand = false;
2102
2103         ibwc->byte_len = 0;
2104         ibwc->qp = &qp->ibqp;
2105         ibwc->status = ocrdma_to_ibwc_err(status);
2106
2107         ocrdma_flush_qp(qp);
2108         ocrdma_qp_state_machine(qp, IB_QPS_ERR, NULL);
2109
2110         /* if wqe/rqe pending for which cqe needs to be returned,
2111          * trigger inflating it.
2112          */
2113         if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) {
2114                 expand = true;
2115                 ocrdma_set_cqe_status_flushed(qp, cqe);
2116         }
2117         return expand;
2118 }
2119
2120 static int ocrdma_update_err_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2121                                   struct ocrdma_qp *qp, int status)
2122 {
2123         ibwc->opcode = IB_WC_RECV;
2124         ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2125         ocrdma_hwq_inc_tail(&qp->rq);
2126
2127         return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2128 }
2129
2130 static int ocrdma_update_err_scqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2131                                   struct ocrdma_qp *qp, int status)
2132 {
2133         ocrdma_update_wc(qp, ibwc, qp->sq.tail);
2134         ocrdma_hwq_inc_tail(&qp->sq);
2135
2136         return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2137 }
2138
2139
2140 static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp,
2141                                  struct ocrdma_cqe *cqe, struct ib_wc *ibwc,
2142                                  bool *polled, bool *stop)
2143 {
2144         bool expand;
2145         int status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2146                 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2147
2148         /* when hw sq is empty, but rq is not empty, so we continue
2149          * to keep the cqe in order to get the cq event again.
2150          */
2151         if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) {
2152                 /* when cq for rq and sq is same, it is safe to return
2153                  * flush cqe for RQEs.
2154                  */
2155                 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2156                         *polled = true;
2157                         status = OCRDMA_CQE_WR_FLUSH_ERR;
2158                         expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
2159                 } else {
2160                         /* stop processing further cqe as this cqe is used for
2161                          * triggering cq event on buddy cq of RQ.
2162                          * When QP is destroyed, this cqe will be removed
2163                          * from the cq's hardware q.
2164                          */
2165                         *polled = false;
2166                         *stop = true;
2167                         expand = false;
2168                 }
2169         } else {
2170                 *polled = true;
2171                 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2172         }
2173         return expand;
2174 }
2175
2176 static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp,
2177                                      struct ocrdma_cqe *cqe,
2178                                      struct ib_wc *ibwc, bool *polled)
2179 {
2180         bool expand = false;
2181         int tail = qp->sq.tail;
2182         u32 wqe_idx;
2183
2184         if (!qp->wqe_wr_id_tbl[tail].signaled) {
2185                 *polled = false;    /* WC cannot be consumed yet */
2186         } else {
2187                 ibwc->status = IB_WC_SUCCESS;
2188                 ibwc->wc_flags = 0;
2189                 ibwc->qp = &qp->ibqp;
2190                 ocrdma_update_wc(qp, ibwc, tail);
2191                 *polled = true;
2192         }
2193         wqe_idx = le32_to_cpu(cqe->wq.wqeidx) & OCRDMA_CQE_WQEIDX_MASK;
2194         if (tail != wqe_idx)
2195                 expand = true; /* Coalesced CQE can't be consumed yet */
2196
2197         ocrdma_hwq_inc_tail(&qp->sq);
2198         return expand;
2199 }
2200
2201 static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2202                              struct ib_wc *ibwc, bool *polled, bool *stop)
2203 {
2204         int status;
2205         bool expand;
2206
2207         status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2208                 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2209
2210         if (status == OCRDMA_CQE_SUCCESS)
2211                 expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled);
2212         else
2213                 expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop);
2214         return expand;
2215 }
2216
2217 static int ocrdma_update_ud_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe)
2218 {
2219         int status;
2220
2221         status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2222                 OCRDMA_CQE_UD_STATUS_MASK) >> OCRDMA_CQE_UD_STATUS_SHIFT;
2223         ibwc->src_qp = le32_to_cpu(cqe->flags_status_srcqpn) &
2224                                                 OCRDMA_CQE_SRCQP_MASK;
2225         ibwc->pkey_index = le32_to_cpu(cqe->ud.rxlen_pkey) &
2226                                                 OCRDMA_CQE_PKEY_MASK;
2227         ibwc->wc_flags = IB_WC_GRH;
2228         ibwc->byte_len = (le32_to_cpu(cqe->ud.rxlen_pkey) >>
2229                                         OCRDMA_CQE_UD_XFER_LEN_SHIFT);
2230         return status;
2231 }
2232
2233 static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc,
2234                                        struct ocrdma_cqe *cqe,
2235                                        struct ocrdma_qp *qp)
2236 {
2237         unsigned long flags;
2238         struct ocrdma_srq *srq;
2239         u32 wqe_idx;
2240
2241         srq = get_ocrdma_srq(qp->ibqp.srq);
2242         wqe_idx = le32_to_cpu(cqe->rq.buftag_qpn) >> OCRDMA_CQE_BUFTAG_SHIFT;
2243         ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx];
2244         spin_lock_irqsave(&srq->q_lock, flags);
2245         ocrdma_srq_toggle_bit(srq, wqe_idx);
2246         spin_unlock_irqrestore(&srq->q_lock, flags);
2247         ocrdma_hwq_inc_tail(&srq->rq);
2248 }
2249
2250 static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2251                                 struct ib_wc *ibwc, bool *polled, bool *stop,
2252                                 int status)
2253 {
2254         bool expand;
2255
2256         /* when hw_rq is empty, but wq is not empty, so continue
2257          * to keep the cqe to get the cq event again.
2258          */
2259         if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) {
2260                 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2261                         *polled = true;
2262                         status = OCRDMA_CQE_WR_FLUSH_ERR;
2263                         expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2264                 } else {
2265                         *polled = false;
2266                         *stop = true;
2267                         expand = false;
2268                 }
2269         } else {
2270                 *polled = true;
2271                 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
2272         }
2273         return expand;
2274 }
2275
2276 static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp,
2277                                      struct ocrdma_cqe *cqe, struct ib_wc *ibwc)
2278 {
2279         ibwc->opcode = IB_WC_RECV;
2280         ibwc->qp = &qp->ibqp;
2281         ibwc->status = IB_WC_SUCCESS;
2282
2283         if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI)
2284                 ocrdma_update_ud_rcqe(ibwc, cqe);
2285         else
2286                 ibwc->byte_len = le32_to_cpu(cqe->rq.rxlen);
2287
2288         if (is_cqe_imm(cqe)) {
2289                 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));
2290                 ibwc->wc_flags |= IB_WC_WITH_IMM;
2291         } else if (is_cqe_wr_imm(cqe)) {
2292                 ibwc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2293                 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));
2294                 ibwc->wc_flags |= IB_WC_WITH_IMM;
2295         } else if (is_cqe_invalidated(cqe)) {
2296                 ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt);
2297                 ibwc->wc_flags |= IB_WC_WITH_INVALIDATE;
2298         }
2299         if (qp->ibqp.srq)
2300                 ocrdma_update_free_srq_cqe(ibwc, cqe, qp);
2301         else {
2302                 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2303                 ocrdma_hwq_inc_tail(&qp->rq);
2304         }
2305 }
2306
2307 static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2308                              struct ib_wc *ibwc, bool *polled, bool *stop)
2309 {
2310         int status;
2311         bool expand = false;
2312
2313         ibwc->wc_flags = 0;
2314         if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI)
2315                 status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2316                                         OCRDMA_CQE_UD_STATUS_MASK) >>
2317                                         OCRDMA_CQE_UD_STATUS_SHIFT;
2318         else
2319                 status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2320                              OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2321
2322         if (status == OCRDMA_CQE_SUCCESS) {
2323                 *polled = true;
2324                 ocrdma_poll_success_rcqe(qp, cqe, ibwc);
2325         } else {
2326                 expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop,
2327                                               status);
2328         }
2329         return expand;
2330 }
2331
2332 static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe,
2333                                    u16 cur_getp)
2334 {
2335         if (cq->phase_change) {
2336                 if (cur_getp == 0)
2337                         cq->phase = (~cq->phase & OCRDMA_CQE_VALID);
2338         } else
2339                 /* clear valid bit */
2340                 cqe->flags_status_srcqpn = 0;
2341 }
2342
2343 static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries,
2344                             struct ib_wc *ibwc)
2345 {
2346         u16 qpn = 0;
2347         int i = 0;
2348         bool expand = false;
2349         int polled_hw_cqes = 0;
2350         struct ocrdma_qp *qp = NULL;
2351         struct ocrdma_dev *dev = cq->dev;
2352         struct ocrdma_cqe *cqe;
2353         u16 cur_getp; bool polled = false; bool stop = false;
2354
2355         cur_getp = cq->getp;
2356         while (num_entries) {
2357                 cqe = cq->va + cur_getp;
2358                 /* check whether valid cqe or not */
2359                 if (!is_cqe_valid(cq, cqe))
2360                         break;
2361                 qpn = (le32_to_cpu(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK);
2362                 /* ignore discarded cqe */
2363                 if (qpn == 0)
2364                         goto skip_cqe;
2365                 qp = dev->qp_tbl[qpn];
2366                 BUG_ON(qp == NULL);
2367
2368                 if (is_cqe_for_sq(cqe)) {
2369                         expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled,
2370                                                   &stop);
2371                 } else {
2372                         expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled,
2373                                                   &stop);
2374                 }
2375                 if (expand)
2376                         goto expand_cqe;
2377                 if (stop)
2378                         goto stop_cqe;
2379                 /* clear qpn to avoid duplicate processing by discard_cqe() */
2380                 cqe->cmn.qpn = 0;
2381 skip_cqe:
2382                 polled_hw_cqes += 1;
2383                 cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
2384                 ocrdma_change_cq_phase(cq, cqe, cur_getp);
2385 expand_cqe:
2386                 if (polled) {
2387                         num_entries -= 1;
2388                         i += 1;
2389                         ibwc = ibwc + 1;
2390                         polled = false;
2391                 }
2392         }
2393 stop_cqe:
2394         cq->getp = cur_getp;
2395         if (polled_hw_cqes || expand || stop) {
2396                 ocrdma_ring_cq_db(dev, cq->id, cq->armed, cq->solicited,
2397                                   polled_hw_cqes);
2398         }
2399         return i;
2400 }
2401
2402 /* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */
2403 static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries,
2404                               struct ocrdma_qp *qp, struct ib_wc *ibwc)
2405 {
2406         int err_cqes = 0;
2407
2408         while (num_entries) {
2409                 if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp))
2410                         break;
2411                 if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) {
2412                         ocrdma_update_wc(qp, ibwc, qp->sq.tail);
2413                         ocrdma_hwq_inc_tail(&qp->sq);
2414                 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) {
2415                         ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2416                         ocrdma_hwq_inc_tail(&qp->rq);
2417                 } else
2418                         return err_cqes;
2419                 ibwc->byte_len = 0;
2420                 ibwc->status = IB_WC_WR_FLUSH_ERR;
2421                 ibwc = ibwc + 1;
2422                 err_cqes += 1;
2423                 num_entries -= 1;
2424         }
2425         return err_cqes;
2426 }
2427
2428 int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
2429 {
2430         int cqes_to_poll = num_entries;
2431         struct ocrdma_cq *cq = NULL;
2432         unsigned long flags;
2433         struct ocrdma_dev *dev;
2434         int num_os_cqe = 0, err_cqes = 0;
2435         struct ocrdma_qp *qp;
2436
2437         cq = get_ocrdma_cq(ibcq);
2438         dev = cq->dev;
2439
2440         /* poll cqes from adapter CQ */
2441         spin_lock_irqsave(&cq->cq_lock, flags);
2442         num_os_cqe = ocrdma_poll_hwcq(cq, cqes_to_poll, wc);
2443         spin_unlock_irqrestore(&cq->cq_lock, flags);
2444         cqes_to_poll -= num_os_cqe;
2445
2446         if (cqes_to_poll) {
2447                 wc = wc + num_os_cqe;
2448                 /* adapter returns single error cqe when qp moves to
2449                  * error state. So insert error cqes with wc_status as
2450                  * FLUSHED for pending WQEs and RQEs of QP's SQ and RQ
2451                  * respectively which uses this CQ.
2452                  */
2453                 spin_lock_irqsave(&dev->flush_q_lock, flags);
2454                 list_for_each_entry(qp, &cq->sq_head, sq_entry) {
2455                         if (cqes_to_poll == 0)
2456                                 break;
2457                         err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc);
2458                         cqes_to_poll -= err_cqes;
2459                         num_os_cqe += err_cqes;
2460                         wc = wc + err_cqes;
2461                 }
2462                 spin_unlock_irqrestore(&dev->flush_q_lock, flags);
2463         }
2464         return num_os_cqe;
2465 }
2466
2467 int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags)
2468 {
2469         struct ocrdma_cq *cq;
2470         unsigned long flags;
2471         struct ocrdma_dev *dev;
2472         u16 cq_id;
2473         u16 cur_getp;
2474         struct ocrdma_cqe *cqe;
2475
2476         cq = get_ocrdma_cq(ibcq);
2477         cq_id = cq->id;
2478         dev = cq->dev;
2479
2480         spin_lock_irqsave(&cq->cq_lock, flags);
2481         if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED)
2482                 cq->armed = true;
2483         if (cq_flags & IB_CQ_SOLICITED)
2484                 cq->solicited = true;
2485
2486         cur_getp = cq->getp;
2487         cqe = cq->va + cur_getp;
2488
2489         /* check whether any valid cqe exist or not, if not then safe to
2490          * arm. If cqe is not yet consumed, then let it get consumed and then
2491          * we arm it to avoid false interrupts.
2492          */
2493         if (!is_cqe_valid(cq, cqe) || cq->arm_needed) {
2494                 cq->arm_needed = false;
2495                 ocrdma_ring_cq_db(dev, cq_id, cq->armed, cq->solicited, 0);
2496         }
2497         spin_unlock_irqrestore(&cq->cq_lock, flags);
2498         return 0;
2499 }