LXR linux/drivers/net/ethernet/mellanox/mlx5/core/alloc.c

   1/*
   2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and/or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32
  33#include <linux/errno.h>
  34#include <linux/slab.h>
  35#include <linux/mm.h>
  36#include <linux/export.h>
  37#include <linux/bitmap.h>
  38#include <linux/dma-mapping.h>
  39#include <linux/vmalloc.h>
  40#include <linux/mlx5/driver.h>
  41
  42#include "mlx5_core.h"
  43
  44struct mlx5_db_pgdir {
  45        struct list_head        list;
  46        unsigned long          *bitmap;
  47        __be32                 *db_page;
  48        dma_addr_t              db_dma;
  49};
  50
  51/* Handling for queue buffers -- we allocate a bunch of memory and
  52 * register it in a memory region at HCA virtual address 0.
  53 */
  54
  55static void *mlx5_dma_zalloc_coherent_node(struct mlx5_core_dev *dev,
  56                                           size_t size, dma_addr_t *dma_handle,
  57                                           int node)
  58{
  59        struct device *device = mlx5_core_dma_dev(dev);
  60        struct mlx5_priv *priv = &dev->priv;
  61        int original_node;
  62        void *cpu_handle;
  63
  64        mutex_lock(&priv->alloc_mutex);
  65        original_node = dev_to_node(device);
  66        set_dev_node(device, node);
  67        cpu_handle = dma_alloc_coherent(device, size, dma_handle,
  68                                        GFP_KERNEL);
  69        set_dev_node(device, original_node);
  70        mutex_unlock(&priv->alloc_mutex);
  71        return cpu_handle;
  72}
  73
  74static int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size,
  75                               struct mlx5_frag_buf *buf, int node)
  76{
  77        dma_addr_t t;
  78
  79        buf->size = size;
  80        buf->npages       = 1;
  81        buf->page_shift   = (u8)get_order(size) + PAGE_SHIFT;
  82
  83        buf->frags = kzalloc(sizeof(*buf->frags), GFP_KERNEL);
  84        if (!buf->frags)
  85                return -ENOMEM;
  86
  87        buf->frags->buf   = mlx5_dma_zalloc_coherent_node(dev, size,
  88                                                          &t, node);
  89        if (!buf->frags->buf)
  90                goto err_out;
  91
  92        buf->frags->map = t;
  93
  94        while (t & ((1 << buf->page_shift) - 1)) {
  95                --buf->page_shift;
  96                buf->npages *= 2;
  97        }
  98
  99        return 0;
 100err_out:
 101        kfree(buf->frags);
 102        return -ENOMEM;
 103}
 104
 105int mlx5_buf_alloc(struct mlx5_core_dev *dev,
 106                   int size, struct mlx5_frag_buf *buf)
 107{
 108        return mlx5_buf_alloc_node(dev, size, buf, dev->priv.numa_node);
 109}
 110EXPORT_SYMBOL(mlx5_buf_alloc);
 111
 112void mlx5_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf)
 113{
 114        dma_free_coherent(mlx5_core_dma_dev(dev), buf->size, buf->frags->buf,
 115                          buf->frags->map);
 116
 117        kfree(buf->frags);
 118}
 119EXPORT_SYMBOL_GPL(mlx5_buf_free);
 120
 121int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size,
 122                             struct mlx5_frag_buf *buf, int node)
 123{
 124        int i;
 125
 126        buf->size = size;
 127        buf->npages = DIV_ROUND_UP(size, PAGE_SIZE);
 128        buf->page_shift = PAGE_SHIFT;
 129        buf->frags = kcalloc(buf->npages, sizeof(struct mlx5_buf_list),
 130                             GFP_KERNEL);
 131        if (!buf->frags)
 132                goto err_out;
 133
 134        for (i = 0; i < buf->npages; i++) {
 135                struct mlx5_buf_list *frag = &buf->frags[i];
 136                int frag_sz = min_t(int, size, PAGE_SIZE);
 137
 138                frag->buf = mlx5_dma_zalloc_coherent_node(dev, frag_sz,
 139                                                          &frag->map, node);
 140                if (!frag->buf)
 141                        goto err_free_buf;
 142                if (frag->map & ((1 << buf->page_shift) - 1)) {
 143                        dma_free_coherent(mlx5_core_dma_dev(dev), frag_sz,
 144                                          buf->frags[i].buf, buf->frags[i].map);
 145                        mlx5_core_warn(dev, "unexpected map alignment: %pad, page_shift=%d\n",
 146                                       &frag->map, buf->page_shift);
 147                        goto err_free_buf;
 148                }
 149                size -= frag_sz;
 150        }
 151
 152        return 0;
 153
 154err_free_buf:
 155        while (i--)
 156                dma_free_coherent(mlx5_core_dma_dev(dev), PAGE_SIZE, buf->frags[i].buf,
 157                                  buf->frags[i].map);
 158        kfree(buf->frags);
 159err_out:
 160        return -ENOMEM;
 161}
 162EXPORT_SYMBOL_GPL(mlx5_frag_buf_alloc_node);
 163
 164void mlx5_frag_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf)
 165{
 166        int size = buf->size;
 167        int i;
 168
 169        for (i = 0; i < buf->npages; i++) {
 170                int frag_sz = min_t(int, size, PAGE_SIZE);
 171
 172                dma_free_coherent(mlx5_core_dma_dev(dev), frag_sz, buf->frags[i].buf,
 173                                  buf->frags[i].map);
 174                size -= frag_sz;
 175        }
 176        kfree(buf->frags);
 177}
 178EXPORT_SYMBOL_GPL(mlx5_frag_buf_free);
 179
 180static struct mlx5_db_pgdir *mlx5_alloc_db_pgdir(struct mlx5_core_dev *dev,
 181                                                 int node)
 182{
 183        u32 db_per_page = PAGE_SIZE / cache_line_size();
 184        struct mlx5_db_pgdir *pgdir;
 185
 186        pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL);
 187        if (!pgdir)
 188                return NULL;
 189
 190        pgdir->bitmap = bitmap_zalloc(db_per_page, GFP_KERNEL);
 191        if (!pgdir->bitmap) {
 192                kfree(pgdir);
 193                return NULL;
 194        }
 195
 196        bitmap_fill(pgdir->bitmap, db_per_page);
 197
 198        pgdir->db_page = mlx5_dma_zalloc_coherent_node(dev, PAGE_SIZE,
 199                                                       &pgdir->db_dma, node);
 200        if (!pgdir->db_page) {
 201                bitmap_free(pgdir->bitmap);
 202                kfree(pgdir);
 203                return NULL;
 204        }
 205
 206        return pgdir;
 207}
 208
 209static int mlx5_alloc_db_from_pgdir(struct mlx5_db_pgdir *pgdir,
 210                                    struct mlx5_db *db)
 211{
 212        u32 db_per_page = PAGE_SIZE / cache_line_size();
 213        int offset;
 214        int i;
 215
 216        i = find_first_bit(pgdir->bitmap, db_per_page);
 217        if (i >= db_per_page)
 218                return -ENOMEM;
 219
 220        __clear_bit(i, pgdir->bitmap);
 221
 222        db->u.pgdir = pgdir;
 223        db->index   = i;
 224        offset = db->index * cache_line_size();
 225        db->db      = pgdir->db_page + offset / sizeof(*pgdir->db_page);
 226        db->dma     = pgdir->db_dma  + offset;
 227
 228        db->db[0] = 0;
 229        db->db[1] = 0;
 230
 231        return 0;
 232}
 233
 234int mlx5_db_alloc_node(struct mlx5_core_dev *dev, struct mlx5_db *db, int node)
 235{
 236        struct mlx5_db_pgdir *pgdir;
 237        int ret = 0;
 238
 239        mutex_lock(&dev->priv.pgdir_mutex);
 240
 241        list_for_each_entry(pgdir, &dev->priv.pgdir_list, list)
 242                if (!mlx5_alloc_db_from_pgdir(pgdir, db))
 243                        goto out;
 244
 245        pgdir = mlx5_alloc_db_pgdir(dev, node);
 246        if (!pgdir) {
 247                ret = -ENOMEM;
 248                goto out;
 249        }
 250
 251        list_add(&pgdir->list, &dev->priv.pgdir_list);
 252
 253        /* This should never fail -- we just allocated an empty page: */
 254        WARN_ON(mlx5_alloc_db_from_pgdir(pgdir, db));
 255
 256out:
 257        mutex_unlock(&dev->priv.pgdir_mutex);
 258
 259        return ret;
 260}
 261EXPORT_SYMBOL_GPL(mlx5_db_alloc_node);
 262
 263int mlx5_db_alloc(struct mlx5_core_dev *dev, struct mlx5_db *db)
 264{
 265        return mlx5_db_alloc_node(dev, db, dev->priv.numa_node);
 266}
 267EXPORT_SYMBOL_GPL(mlx5_db_alloc);
 268
 269void mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db)
 270{
 271        u32 db_per_page = PAGE_SIZE / cache_line_size();
 272
 273        mutex_lock(&dev->priv.pgdir_mutex);
 274
 275        __set_bit(db->index, db->u.pgdir->bitmap);
 276
 277        if (bitmap_full(db->u.pgdir->bitmap, db_per_page)) {
 278                dma_free_coherent(mlx5_core_dma_dev(dev), PAGE_SIZE,
 279                                  db->u.pgdir->db_page, db->u.pgdir->db_dma);
 280                list_del(&db->u.pgdir->list);
 281                bitmap_free(db->u.pgdir->bitmap);
 282                kfree(db->u.pgdir);
 283        }
 284
 285        mutex_unlock(&dev->priv.pgdir_mutex);
 286}
 287EXPORT_SYMBOL_GPL(mlx5_db_free);
 288
 289void mlx5_fill_page_array(struct mlx5_frag_buf *buf, __be64 *pas)
 290{
 291        u64 addr;
 292        int i;
 293
 294        for (i = 0; i < buf->npages; i++) {
 295                addr = buf->frags->map + (i << buf->page_shift);
 296
 297                pas[i] = cpu_to_be64(addr);
 298        }
 299}
 300EXPORT_SYMBOL_GPL(mlx5_fill_page_array);
 301
 302void mlx5_fill_page_frag_array_perm(struct mlx5_frag_buf *buf, __be64 *pas, u8 perm)
 303{
 304        int i;
 305
 306        WARN_ON(perm & 0xfc);
 307        for (i = 0; i < buf->npages; i++)
 308                pas[i] = cpu_to_be64(buf->frags[i].map | perm);
 309}
 310EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array_perm);
 311
 312void mlx5_fill_page_frag_array(struct mlx5_frag_buf *buf, __be64 *pas)
 313{
 314        mlx5_fill_page_frag_array_perm(buf, pas, 0);
 315}
 316EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array);
 317