drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c

   1 /*
   2  * Copyright (c) 2013, Mellanox Technologies inc.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <asm-generic/kmap_types.h>
  34 #include <linux/kernel.h>
  35 #include <linux/module.h>
  36 #include <linux/mlx5/driver.h>
  37 #include <linux/mlx5/cmd.h>
  38 #include "mlx5_core.h"
  39
  40 enum {
  41         MLX5_PAGES_CANT_GIVE    = 0,
  42         MLX5_PAGES_GIVE         = 1,
  43         MLX5_PAGES_TAKE         = 2
  44 };
  45
  46 struct mlx5_pages_req {
  47         struct mlx5_core_dev *dev;
  48         u32     func_id;
  49         s16     npages;
  50         struct work_struct work;
  51 };
  52
  53 struct fw_page {
  54         struct rb_node  rb_node;
  55         u64             addr;
  56         struct page     *page;
  57         u16             func_id;
  58 };
  59
  60 struct mlx5_query_pages_inbox {
  61         struct mlx5_inbox_hdr   hdr;
  62         u8                      rsvd[8];
  63 };
  64
  65 struct mlx5_query_pages_outbox {
  66         struct mlx5_outbox_hdr  hdr;
  67         __be16                  num_boot_pages;
  68         __be16                  func_id;
  69         __be16                  init_pages;
  70         __be16                  num_pages;
  71 };
  72
  73 struct mlx5_manage_pages_inbox {
  74         struct mlx5_inbox_hdr   hdr;
  75         __be16                  rsvd0;
  76         __be16                  func_id;
  77         __be16                  rsvd1;
  78         __be16                  num_entries;
  79         u8                      rsvd2[16];
  80         __be64                  pas[0];
  81 };
  82
  83 struct mlx5_manage_pages_outbox {
  84         struct mlx5_outbox_hdr  hdr;
  85         u8                      rsvd0[2];
  86         __be16                  num_entries;
  87         u8                      rsvd1[20];
  88         __be64                  pas[0];
  89 };
  90
  91 static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u16 func_id)
  92 {
  93         struct rb_root *root = &dev->priv.page_root;
  94         struct rb_node **new = &root->rb_node;
  95         struct rb_node *parent = NULL;
  96         struct fw_page *nfp;
  97         struct fw_page *tfp;
  98
  99         while (*new) {
 100                 parent = *new;
 101                 tfp = rb_entry(parent, struct fw_page, rb_node);
 102                 if (tfp->addr < addr)
 103                         new = &parent->rb_left;
 104                 else if (tfp->addr > addr)
 105                         new = &parent->rb_right;
 106                 else
 107                         return -EEXIST;
 108         }
 109
 110         nfp = kmalloc(sizeof(*nfp), GFP_KERNEL);
 111         if (!nfp)
 112                 return -ENOMEM;
 113
 114         nfp->addr = addr;
 115         nfp->page = page;
 116         nfp->func_id = func_id;
 117
 118         rb_link_node(&nfp->rb_node, parent, new);
 119         rb_insert_color(&nfp->rb_node, root);
 120
 121         return 0;
 122 }
 123
 124 static struct page *remove_page(struct mlx5_core_dev *dev, u64 addr)
 125 {
 126         struct rb_root *root = &dev->priv.page_root;
 127         struct rb_node *tmp = root->rb_node;
 128         struct page *result = NULL;
 129         struct fw_page *tfp;
 130
 131         while (tmp) {
 132                 tfp = rb_entry(tmp, struct fw_page, rb_node);
 133                 if (tfp->addr < addr) {
 134                         tmp = tmp->rb_left;
 135                 } else if (tfp->addr > addr) {
 136                         tmp = tmp->rb_right;
 137                 } else {
 138                         rb_erase(&tfp->rb_node, root);
 139                         result = tfp->page;
 140                         kfree(tfp);
 141                         break;
 142                 }
 143         }
 144
 145         return result;
 146 }
 147
 148 static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id,
 149                                 s16 *pages, s16 *init_pages, u16 *boot_pages)
 150 {
 151         struct mlx5_query_pages_inbox   in;
 152         struct mlx5_query_pages_outbox  out;
 153         int err;
 154
 155         memset(&in, 0, sizeof(in));
 156         memset(&out, 0, sizeof(out));
 157         in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_PAGES);
 158         err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
 159         if (err)
 160                 return err;
 161
 162         if (out.hdr.status)
 163                 return mlx5_cmd_status_to_err(&out.hdr);
 164
 165         if (pages)
 166                 *pages = be16_to_cpu(out.num_pages);
 167
 168         if (init_pages)
 169                 *init_pages = be16_to_cpu(out.init_pages);
 170
 171         if (boot_pages)
 172                 *boot_pages = be16_to_cpu(out.num_boot_pages);
 173
 174         *func_id = be16_to_cpu(out.func_id);
 175
 176         return err;
 177 }
 178
 179 static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
 180                       int notify_fail)
 181 {
 182         struct mlx5_manage_pages_inbox *in;
 183         struct mlx5_manage_pages_outbox out;
 184         struct page *page;
 185         int inlen;
 186         u64 addr;
 187         int err;
 188         int i;
 189
 190         inlen = sizeof(*in) + npages * sizeof(in->pas[0]);
 191         in = mlx5_vzalloc(inlen);
 192         if (!in) {
 193                 mlx5_core_warn(dev, "vzalloc failed %d\n", inlen);
 194                 return -ENOMEM;
 195         }
 196         memset(&out, 0, sizeof(out));
 197
 198         for (i = 0; i < npages; i++) {
 199                 page = alloc_page(GFP_HIGHUSER);
 200                 if (!page) {
 201                         err = -ENOMEM;
 202                         mlx5_core_warn(dev, "failed to allocate page\n");
 203                         goto out_alloc;
 204                 }
 205                 addr = dma_map_page(&dev->pdev->dev, page, 0,
 206                                     PAGE_SIZE, DMA_BIDIRECTIONAL);
 207                 if (dma_mapping_error(&dev->pdev->dev, addr)) {
 208                         mlx5_core_warn(dev, "failed dma mapping page\n");
 209                         __free_page(page);
 210                         err = -ENOMEM;
 211                         goto out_alloc;
 212                 }
 213                 err = insert_page(dev, addr, page, func_id);
 214                 if (err) {
 215                         mlx5_core_err(dev, "failed to track allocated page\n");
 216                         dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
 217                         __free_page(page);
 218                         err = -ENOMEM;
 219                         goto out_alloc;
 220                 }
 221                 in->pas[i] = cpu_to_be64(addr);
 222         }
 223
 224         in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
 225         in->hdr.opmod = cpu_to_be16(MLX5_PAGES_GIVE);
 226         in->func_id = cpu_to_be16(func_id);
 227         in->num_entries = cpu_to_be16(npages);
 228         err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out));
 229         mlx5_core_dbg(dev, "err %d\n", err);
 230         if (err) {
 231                 mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n", func_id, npages, err);
 232                 goto out_alloc;
 233         }
 234         dev->priv.fw_pages += npages;
 235
 236         if (out.hdr.status) {
 237                 err = mlx5_cmd_status_to_err(&out.hdr);
 238                 if (err) {
 239                         mlx5_core_warn(dev, "func_id 0x%x, npages %d, status %d\n", func_id, npages, out.hdr.status);
 240                         goto out_alloc;
 241                 }
 242         }
 243
 244         mlx5_core_dbg(dev, "err %d\n", err);
 245
 246         goto out_free;
 247
 248 out_alloc:
 249         if (notify_fail) {
 250                 memset(in, 0, inlen);
 251                 memset(&out, 0, sizeof(out));
 252                 in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
 253                 in->hdr.opmod = cpu_to_be16(MLX5_PAGES_CANT_GIVE);
 254                 if (mlx5_cmd_exec(dev, in, sizeof(*in), &out, sizeof(out)))
 255                         mlx5_core_warn(dev, "\n");
 256         }
 257         for (i--; i >= 0; i--) {
 258                 addr = be64_to_cpu(in->pas[i]);
 259                 page = remove_page(dev, addr);
 260                 if (!page) {
 261                         mlx5_core_err(dev, "BUG: can't remove page at addr 0x%llx\n",
 262                                       addr);
 263                         continue;
 264                 }
 265                 dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
 266                 __free_page(page);
 267         }
 268
 269 out_free:
 270         mlx5_vfree(in);
 271         return err;
 272 }
 273
 274 static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages,
 275                          int *nclaimed)
 276 {
 277         struct mlx5_manage_pages_inbox   in;
 278         struct mlx5_manage_pages_outbox *out;
 279         struct page *page;
 280         int num_claimed;
 281         int outlen;
 282         u64 addr;
 283         int err;
 284         int i;
 285
 286         memset(&in, 0, sizeof(in));
 287         outlen = sizeof(*out) + npages * sizeof(out->pas[0]);
 288         out = mlx5_vzalloc(outlen);
 289         if (!out)
 290                 return -ENOMEM;
 291
 292         in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
 293         in.hdr.opmod = cpu_to_be16(MLX5_PAGES_TAKE);
 294         in.func_id = cpu_to_be16(func_id);
 295         in.num_entries = cpu_to_be16(npages);
 296         mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen);
 297         err = mlx5_cmd_exec(dev, &in, sizeof(in), out, outlen);
 298         if (err) {
 299                 mlx5_core_err(dev, "failed recliaming pages\n");
 300                 goto out_free;
 301         }
 302         dev->priv.fw_pages -= npages;
 303
 304         if (out->hdr.status) {
 305                 err = mlx5_cmd_status_to_err(&out->hdr);
 306                 goto out_free;
 307         }
 308
 309         num_claimed = be16_to_cpu(out->num_entries);
 310         if (nclaimed)
 311                 *nclaimed = num_claimed;
 312
 313         for (i = 0; i < num_claimed; i++) {
 314                 addr = be64_to_cpu(out->pas[i]);
 315                 page = remove_page(dev, addr);
 316                 if (!page) {
 317                         mlx5_core_warn(dev, "FW reported unknown DMA address 0x%llx\n", addr);
 318                 } else {
 319                         dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
 320                         __free_page(page);
 321                 }
 322         }
 323
 324 out_free:
 325         mlx5_vfree(out);
 326         return err;
 327 }
 328
 329 static void pages_work_handler(struct work_struct *work)
 330 {
 331         struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work);
 332         struct mlx5_core_dev *dev = req->dev;
 333         int err = 0;
 334
 335         if (req->npages < 0)
 336                 err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL);
 337         else if (req->npages > 0)
 338                 err = give_pages(dev, req->func_id, req->npages, 1);
 339
 340         if (err)
 341                 mlx5_core_warn(dev, "%s fail %d\n", req->npages < 0 ?
 342                                "reclaim" : "give", err);
 343
 344         kfree(req);
 345 }
 346
 347 void mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id,
 348                                  s16 npages)
 349 {
 350         struct mlx5_pages_req *req;
 351
 352         req = kzalloc(sizeof(*req), GFP_ATOMIC);
 353         if (!req) {
 354                 mlx5_core_warn(dev, "failed to allocate pages request\n");
 355                 return;
 356         }
 357
 358         req->dev = dev;
 359         req->func_id = func_id;
 360         req->npages = npages;
 361         INIT_WORK(&req->work, pages_work_handler);
 362         queue_work(dev->priv.pg_wq, &req->work);
 363 }
 364
 365 int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot)
 366 {
 367         u16 uninitialized_var(boot_pages);
 368         s16 uninitialized_var(init_pages);
 369         u16 uninitialized_var(func_id);
 370         int err;
 371
 372         err = mlx5_cmd_query_pages(dev, &func_id, NULL, &init_pages,
 373                                    &boot_pages);
 374         if (err)
 375                 return err;
 376
 377
 378         mlx5_core_dbg(dev, "requested %d init pages and %d boot pages for func_id 0x%x\n",
 379                       init_pages, boot_pages, func_id);
 380         return give_pages(dev, func_id, boot ? boot_pages : init_pages, 0);
 381 }
 382
 383 static int optimal_reclaimed_pages(void)
 384 {
 385         struct mlx5_cmd_prot_block *block;
 386         struct mlx5_cmd_layout *lay;
 387         int ret;
 388
 389         ret = (sizeof(lay->in) + sizeof(block->data) -
 390                sizeof(struct mlx5_manage_pages_outbox)) / 8;
 391
 392         return ret;
 393 }
 394
 395 int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev)
 396 {
 397         unsigned long end = jiffies + msecs_to_jiffies(5000);
 398         struct fw_page *fwp;
 399         struct rb_node *p;
 400         int err;
 401
 402         do {
 403                 p = rb_first(&dev->priv.page_root);
 404                 if (p) {
 405                         fwp = rb_entry(p, struct fw_page, rb_node);
 406                         err = reclaim_pages(dev, fwp->func_id, optimal_reclaimed_pages(), NULL);
 407                         if (err) {
 408                                 mlx5_core_warn(dev, "failed reclaiming pages (%d)\n", err);
 409                                 return err;
 410                         }
 411                 }
 412                 if (time_after(jiffies, end)) {
 413                         mlx5_core_warn(dev, "FW did not return all pages. giving up...\n");
 414                         break;
 415                 }
 416         } while (p);
 417
 418         return 0;
 419 }
 420
 421 void mlx5_pagealloc_init(struct mlx5_core_dev *dev)
 422 {
 423         dev->priv.page_root = RB_ROOT;
 424 }
 425
 426 void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev)
 427 {
 428         /* nothing */
 429 }
 430
 431 int mlx5_pagealloc_start(struct mlx5_core_dev *dev)
 432 {
 433         dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator");
 434         if (!dev->priv.pg_wq)
 435                 return -ENOMEM;
 436
 437         return 0;
 438 }
 439
 440 void mlx5_pagealloc_stop(struct mlx5_core_dev *dev)
 441 {
 442         destroy_workqueue(dev->priv.pg_wq);
 443 }