linux/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
<<
>>
Prefs
   1/*
   2 * cxgb4_uld.c:Chelsio Upper Layer Driver Interface for T4/T5/T6 SGE management
   3 *
   4 * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved.
   5 *
   6 * This software is available to you under a choice of one of two
   7 * licenses.  You may choose to be licensed under the terms of the GNU
   8 * General Public License (GPL) Version 2, available from the file
   9 * COPYING in the main directory of this source tree, or the
  10 * OpenIB.org BSD license below:
  11 *
  12 *     Redistribution and use in source and binary forms, with or
  13 *     without modification, are permitted provided that the following
  14 *     conditions are met:
  15 *
  16 *      - Redistributions of source code must retain the above
  17 *        copyright notice, this list of conditions and the following
  18 *        disclaimer.
  19 *
  20 *      - Redistributions in binary form must reproduce the above
  21 *        copyright notice, this list of conditions and the following
  22 *        disclaimer in the documentation and/or other materials
  23 *        provided with the distribution.
  24 *
  25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32 * SOFTWARE.
  33 *
  34 *  Written by: Atul Gupta (atul.gupta@chelsio.com)
  35 *  Written by: Hariprasad Shenai (hariprasad@chelsio.com)
  36 */
  37
  38#include <linux/kernel.h>
  39#include <linux/module.h>
  40#include <linux/errno.h>
  41#include <linux/types.h>
  42#include <linux/debugfs.h>
  43#include <linux/export.h>
  44#include <linux/list.h>
  45#include <linux/skbuff.h>
  46#include <linux/pci.h>
  47
  48#include "cxgb4.h"
  49#include "cxgb4_uld.h"
  50#include "t4_regs.h"
  51#include "t4fw_api.h"
  52#include "t4_msg.h"
  53
  54#define for_each_uldrxq(m, i) for (i = 0; i < ((m)->nrxq + (m)->nciq); i++)
  55
  56static int get_msix_idx_from_bmap(struct adapter *adap)
  57{
  58        struct uld_msix_bmap *bmap = &adap->msix_bmap_ulds;
  59        unsigned long flags;
  60        unsigned int msix_idx;
  61
  62        spin_lock_irqsave(&bmap->lock, flags);
  63        msix_idx = find_first_zero_bit(bmap->msix_bmap, bmap->mapsize);
  64        if (msix_idx < bmap->mapsize) {
  65                __set_bit(msix_idx, bmap->msix_bmap);
  66        } else {
  67                spin_unlock_irqrestore(&bmap->lock, flags);
  68                return -ENOSPC;
  69        }
  70
  71        spin_unlock_irqrestore(&bmap->lock, flags);
  72        return msix_idx;
  73}
  74
  75static void free_msix_idx_in_bmap(struct adapter *adap, unsigned int msix_idx)
  76{
  77        struct uld_msix_bmap *bmap = &adap->msix_bmap_ulds;
  78        unsigned long flags;
  79
  80        spin_lock_irqsave(&bmap->lock, flags);
  81         __clear_bit(msix_idx, bmap->msix_bmap);
  82        spin_unlock_irqrestore(&bmap->lock, flags);
  83}
  84
  85/* Flush the aggregated lro sessions */
  86static void uldrx_flush_handler(struct sge_rspq *q)
  87{
  88        struct adapter *adap = q->adap;
  89
  90        if (adap->uld[q->uld].lro_flush)
  91                adap->uld[q->uld].lro_flush(&q->lro_mgr);
  92}
  93
  94/**
  95 *      uldrx_handler - response queue handler for ULD queues
  96 *      @q: the response queue that received the packet
  97 *      @rsp: the response queue descriptor holding the offload message
  98 *      @gl: the gather list of packet fragments
  99 *
 100 *      Deliver an ingress offload packet to a ULD.  All processing is done by
 101 *      the ULD, we just maintain statistics.
 102 */
 103static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
 104                         const struct pkt_gl *gl)
 105{
 106        struct adapter *adap = q->adap;
 107        struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
 108        int ret;
 109
 110        /* FW can send CPLs encapsulated in a CPL_FW4_MSG */
 111        if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
 112            ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
 113                rsp += 2;
 114
 115        if (q->flush_handler)
 116                ret = adap->uld[q->uld].lro_rx_handler(adap->uld[q->uld].handle,
 117                                rsp, gl, &q->lro_mgr,
 118                                &q->napi);
 119        else
 120                ret = adap->uld[q->uld].rx_handler(adap->uld[q->uld].handle,
 121                                rsp, gl);
 122
 123        if (ret) {
 124                rxq->stats.nomem++;
 125                return -1;
 126        }
 127
 128        if (!gl)
 129                rxq->stats.imm++;
 130        else if (gl == CXGB4_MSG_AN)
 131                rxq->stats.an++;
 132        else
 133                rxq->stats.pkts++;
 134        return 0;
 135}
 136
 137static int alloc_uld_rxqs(struct adapter *adap,
 138                          struct sge_uld_rxq_info *rxq_info, bool lro)
 139{
 140        struct sge *s = &adap->sge;
 141        unsigned int nq = rxq_info->nrxq + rxq_info->nciq;
 142        struct sge_ofld_rxq *q = rxq_info->uldrxq;
 143        unsigned short *ids = rxq_info->rspq_id;
 144        unsigned int bmap_idx = 0;
 145        unsigned int per_chan;
 146        int i, err, msi_idx, que_idx = 0;
 147
 148        per_chan = rxq_info->nrxq / adap->params.nports;
 149
 150        if (adap->flags & USING_MSIX)
 151                msi_idx = 1;
 152        else
 153                msi_idx = -((int)s->intrq.abs_id + 1);
 154
 155        for (i = 0; i < nq; i++, q++) {
 156                if (i == rxq_info->nrxq) {
 157                        /* start allocation of concentrator queues */
 158                        per_chan = rxq_info->nciq / adap->params.nports;
 159                        que_idx = 0;
 160                }
 161
 162                if (msi_idx >= 0) {
 163                        bmap_idx = get_msix_idx_from_bmap(adap);
 164                        msi_idx = adap->msix_info_ulds[bmap_idx].idx;
 165                }
 166                err = t4_sge_alloc_rxq(adap, &q->rspq, false,
 167                                       adap->port[que_idx++ / per_chan],
 168                                       msi_idx,
 169                                       q->fl.size ? &q->fl : NULL,
 170                                       uldrx_handler,
 171                                       lro ? uldrx_flush_handler : NULL,
 172                                       0);
 173                if (err)
 174                        goto freeout;
 175                if (msi_idx >= 0)
 176                        rxq_info->msix_tbl[i] = bmap_idx;
 177                memset(&q->stats, 0, sizeof(q->stats));
 178                if (ids)
 179                        ids[i] = q->rspq.abs_id;
 180        }
 181        return 0;
 182freeout:
 183        q = rxq_info->uldrxq;
 184        for ( ; i; i--, q++) {
 185                if (q->rspq.desc)
 186                        free_rspq_fl(adap, &q->rspq,
 187                                     q->fl.size ? &q->fl : NULL);
 188        }
 189        return err;
 190}
 191
 192static int
 193setup_sge_queues_uld(struct adapter *adap, unsigned int uld_type, bool lro)
 194{
 195        struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 196        int i, ret = 0;
 197
 198        if (adap->flags & USING_MSIX) {
 199                rxq_info->msix_tbl = kcalloc((rxq_info->nrxq + rxq_info->nciq),
 200                                             sizeof(unsigned short),
 201                                             GFP_KERNEL);
 202                if (!rxq_info->msix_tbl)
 203                        return -ENOMEM;
 204        }
 205
 206        ret = !(!alloc_uld_rxqs(adap, rxq_info, lro));
 207
 208        /* Tell uP to route control queue completions to rdma rspq */
 209        if (adap->flags & FULL_INIT_DONE &&
 210            !ret && uld_type == CXGB4_ULD_RDMA) {
 211                struct sge *s = &adap->sge;
 212                unsigned int cmplqid;
 213                u32 param, cmdop;
 214
 215                cmdop = FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL;
 216                for_each_port(adap, i) {
 217                        cmplqid = rxq_info->uldrxq[i].rspq.cntxt_id;
 218                        param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
 219                                 FW_PARAMS_PARAM_X_V(cmdop) |
 220                                 FW_PARAMS_PARAM_YZ_V(s->ctrlq[i].q.cntxt_id));
 221                        ret = t4_set_params(adap, adap->mbox, adap->pf,
 222                                            0, 1, &param, &cmplqid);
 223                }
 224        }
 225        return ret;
 226}
 227
 228static void t4_free_uld_rxqs(struct adapter *adap, int n,
 229                             struct sge_ofld_rxq *q)
 230{
 231        for ( ; n; n--, q++) {
 232                if (q->rspq.desc)
 233                        free_rspq_fl(adap, &q->rspq,
 234                                     q->fl.size ? &q->fl : NULL);
 235        }
 236}
 237
 238static void free_sge_queues_uld(struct adapter *adap, unsigned int uld_type)
 239{
 240        struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 241
 242        if (adap->flags & FULL_INIT_DONE && uld_type == CXGB4_ULD_RDMA) {
 243                struct sge *s = &adap->sge;
 244                u32 param, cmdop, cmplqid = 0;
 245                int i;
 246
 247                cmdop = FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL;
 248                for_each_port(adap, i) {
 249                        param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
 250                                 FW_PARAMS_PARAM_X_V(cmdop) |
 251                                 FW_PARAMS_PARAM_YZ_V(s->ctrlq[i].q.cntxt_id));
 252                        t4_set_params(adap, adap->mbox, adap->pf,
 253                                      0, 1, &param, &cmplqid);
 254                }
 255        }
 256
 257        if (rxq_info->nciq)
 258                t4_free_uld_rxqs(adap, rxq_info->nciq,
 259                                 rxq_info->uldrxq + rxq_info->nrxq);
 260        t4_free_uld_rxqs(adap, rxq_info->nrxq, rxq_info->uldrxq);
 261        if (adap->flags & USING_MSIX)
 262                kfree(rxq_info->msix_tbl);
 263}
 264
 265static int cfg_queues_uld(struct adapter *adap, unsigned int uld_type,
 266                          const struct cxgb4_uld_info *uld_info)
 267{
 268        struct sge *s = &adap->sge;
 269        struct sge_uld_rxq_info *rxq_info;
 270        int i, nrxq, ciq_size;
 271
 272        rxq_info = kzalloc(sizeof(*rxq_info), GFP_KERNEL);
 273        if (!rxq_info)
 274                return -ENOMEM;
 275
 276        if (adap->flags & USING_MSIX && uld_info->nrxq > s->nqs_per_uld) {
 277                i = s->nqs_per_uld;
 278                rxq_info->nrxq = roundup(i, adap->params.nports);
 279        } else {
 280                i = min_t(int, uld_info->nrxq,
 281                          num_online_cpus());
 282                rxq_info->nrxq = roundup(i, adap->params.nports);
 283        }
 284        if (!uld_info->ciq) {
 285                rxq_info->nciq = 0;
 286        } else  {
 287                if (adap->flags & USING_MSIX)
 288                        rxq_info->nciq = min_t(int, s->nqs_per_uld,
 289                                               num_online_cpus());
 290                else
 291                        rxq_info->nciq = min_t(int, MAX_OFLD_QSETS,
 292                                               num_online_cpus());
 293                rxq_info->nciq = ((rxq_info->nciq / adap->params.nports) *
 294                                  adap->params.nports);
 295                rxq_info->nciq = max_t(int, rxq_info->nciq,
 296                                       adap->params.nports);
 297        }
 298
 299        nrxq = rxq_info->nrxq + rxq_info->nciq; /* total rxq's */
 300        rxq_info->uldrxq = kcalloc(nrxq, sizeof(struct sge_ofld_rxq),
 301                                   GFP_KERNEL);
 302        if (!rxq_info->uldrxq) {
 303                kfree(rxq_info);
 304                return -ENOMEM;
 305        }
 306
 307        rxq_info->rspq_id = kcalloc(nrxq, sizeof(unsigned short), GFP_KERNEL);
 308        if (!rxq_info->rspq_id) {
 309                kfree(rxq_info->uldrxq);
 310                kfree(rxq_info);
 311                return -ENOMEM;
 312        }
 313
 314        for (i = 0; i < rxq_info->nrxq; i++) {
 315                struct sge_ofld_rxq *r = &rxq_info->uldrxq[i];
 316
 317                init_rspq(adap, &r->rspq, 5, 1, uld_info->rxq_size, 64);
 318                r->rspq.uld = uld_type;
 319                r->fl.size = 72;
 320        }
 321
 322        ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids;
 323        if (ciq_size > SGE_MAX_IQ_SIZE) {
 324                dev_warn(adap->pdev_dev, "CIQ size too small for available IQs\n");
 325                ciq_size = SGE_MAX_IQ_SIZE;
 326        }
 327
 328        for (i = rxq_info->nrxq; i < nrxq; i++) {
 329                struct sge_ofld_rxq *r = &rxq_info->uldrxq[i];
 330
 331                init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64);
 332                r->rspq.uld = uld_type;
 333        }
 334
 335        memcpy(rxq_info->name, uld_info->name, IFNAMSIZ);
 336        adap->sge.uld_rxq_info[uld_type] = rxq_info;
 337
 338        return 0;
 339}
 340
 341static void free_queues_uld(struct adapter *adap, unsigned int uld_type)
 342{
 343        struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 344
 345        kfree(rxq_info->rspq_id);
 346        kfree(rxq_info->uldrxq);
 347        kfree(rxq_info);
 348}
 349
 350static int
 351request_msix_queue_irqs_uld(struct adapter *adap, unsigned int uld_type)
 352{
 353        struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 354        int err = 0;
 355        unsigned int idx, bmap_idx;
 356
 357        for_each_uldrxq(rxq_info, idx) {
 358                bmap_idx = rxq_info->msix_tbl[idx];
 359                err = request_irq(adap->msix_info_ulds[bmap_idx].vec,
 360                                  t4_sge_intr_msix, 0,
 361                                  adap->msix_info_ulds[bmap_idx].desc,
 362                                  &rxq_info->uldrxq[idx].rspq);
 363                if (err)
 364                        goto unwind;
 365        }
 366        return 0;
 367unwind:
 368        while (idx-- > 0) {
 369                bmap_idx = rxq_info->msix_tbl[idx];
 370                free_msix_idx_in_bmap(adap, bmap_idx);
 371                free_irq(adap->msix_info_ulds[bmap_idx].vec,
 372                         &rxq_info->uldrxq[idx].rspq);
 373        }
 374        return err;
 375}
 376
 377static void
 378free_msix_queue_irqs_uld(struct adapter *adap, unsigned int uld_type)
 379{
 380        struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 381        unsigned int idx, bmap_idx;
 382
 383        for_each_uldrxq(rxq_info, idx) {
 384                bmap_idx = rxq_info->msix_tbl[idx];
 385
 386                free_msix_idx_in_bmap(adap, bmap_idx);
 387                free_irq(adap->msix_info_ulds[bmap_idx].vec,
 388                         &rxq_info->uldrxq[idx].rspq);
 389        }
 390}
 391
 392static void name_msix_vecs_uld(struct adapter *adap, unsigned int uld_type)
 393{
 394        struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 395        int n = sizeof(adap->msix_info_ulds[0].desc);
 396        unsigned int idx, bmap_idx;
 397
 398        for_each_uldrxq(rxq_info, idx) {
 399                bmap_idx = rxq_info->msix_tbl[idx];
 400
 401                snprintf(adap->msix_info_ulds[bmap_idx].desc, n, "%s-%s%d",
 402                         adap->port[0]->name, rxq_info->name, idx);
 403        }
 404}
 405
 406static void enable_rx(struct adapter *adap, struct sge_rspq *q)
 407{
 408        if (!q)
 409                return;
 410
 411        if (q->handler)
 412                napi_enable(&q->napi);
 413
 414        /* 0-increment GTS to start the timer and enable interrupts */
 415        t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
 416                     SEINTARM_V(q->intr_params) |
 417                     INGRESSQID_V(q->cntxt_id));
 418}
 419
 420static void quiesce_rx(struct adapter *adap, struct sge_rspq *q)
 421{
 422        if (q && q->handler)
 423                napi_disable(&q->napi);
 424}
 425
 426static void enable_rx_uld(struct adapter *adap, unsigned int uld_type)
 427{
 428        struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 429        int idx;
 430
 431        for_each_uldrxq(rxq_info, idx)
 432                enable_rx(adap, &rxq_info->uldrxq[idx].rspq);
 433}
 434
 435static void quiesce_rx_uld(struct adapter *adap, unsigned int uld_type)
 436{
 437        struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 438        int idx;
 439
 440        for_each_uldrxq(rxq_info, idx)
 441                quiesce_rx(adap, &rxq_info->uldrxq[idx].rspq);
 442}
 443
 444static void
 445free_sge_txq_uld(struct adapter *adap, struct sge_uld_txq_info *txq_info)
 446{
 447        int nq = txq_info->ntxq;
 448        int i;
 449
 450        for (i = 0; i < nq; i++) {
 451                struct sge_uld_txq *txq = &txq_info->uldtxq[i];
 452
 453                if (txq && txq->q.desc) {
 454                        tasklet_kill(&txq->qresume_tsk);
 455                        t4_ofld_eq_free(adap, adap->mbox, adap->pf, 0,
 456                                        txq->q.cntxt_id);
 457                        free_tx_desc(adap, &txq->q, txq->q.in_use, false);
 458                        kfree(txq->q.sdesc);
 459                        __skb_queue_purge(&txq->sendq);
 460                        free_txq(adap, &txq->q);
 461                }
 462        }
 463}
 464
 465static int
 466alloc_sge_txq_uld(struct adapter *adap, struct sge_uld_txq_info *txq_info,
 467                  unsigned int uld_type)
 468{
 469        struct sge *s = &adap->sge;
 470        int nq = txq_info->ntxq;
 471        int i, j, err;
 472
 473        j = nq / adap->params.nports;
 474        for (i = 0; i < nq; i++) {
 475                struct sge_uld_txq *txq = &txq_info->uldtxq[i];
 476
 477                txq->q.size = 1024;
 478                err = t4_sge_alloc_uld_txq(adap, txq, adap->port[i / j],
 479                                           s->fw_evtq.cntxt_id, uld_type);
 480                if (err)
 481                        goto freeout;
 482        }
 483        return 0;
 484freeout:
 485        free_sge_txq_uld(adap, txq_info);
 486        return err;
 487}
 488
 489static void
 490release_sge_txq_uld(struct adapter *adap, unsigned int uld_type)
 491{
 492        struct sge_uld_txq_info *txq_info = NULL;
 493        int tx_uld_type = TX_ULD(uld_type);
 494
 495        txq_info = adap->sge.uld_txq_info[tx_uld_type];
 496
 497        if (txq_info && atomic_dec_and_test(&txq_info->users)) {
 498                free_sge_txq_uld(adap, txq_info);
 499                kfree(txq_info->uldtxq);
 500                kfree(txq_info);
 501                adap->sge.uld_txq_info[tx_uld_type] = NULL;
 502        }
 503}
 504
 505static int
 506setup_sge_txq_uld(struct adapter *adap, unsigned int uld_type,
 507                  const struct cxgb4_uld_info *uld_info)
 508{
 509        struct sge_uld_txq_info *txq_info = NULL;
 510        int tx_uld_type, i;
 511
 512        tx_uld_type = TX_ULD(uld_type);
 513        txq_info = adap->sge.uld_txq_info[tx_uld_type];
 514
 515        if ((tx_uld_type == CXGB4_TX_OFLD) && txq_info &&
 516            (atomic_inc_return(&txq_info->users) > 1))
 517                return 0;
 518
 519        txq_info = kzalloc(sizeof(*txq_info), GFP_KERNEL);
 520        if (!txq_info)
 521                return -ENOMEM;
 522
 523        i = min_t(int, uld_info->ntxq, num_online_cpus());
 524        txq_info->ntxq = roundup(i, adap->params.nports);
 525
 526        txq_info->uldtxq = kcalloc(txq_info->ntxq, sizeof(struct sge_uld_txq),
 527                                   GFP_KERNEL);
 528        if (!txq_info->uldtxq) {
 529                kfree(txq_info);
 530                return -ENOMEM;
 531        }
 532
 533        if (alloc_sge_txq_uld(adap, txq_info, tx_uld_type)) {
 534                kfree(txq_info->uldtxq);
 535                kfree(txq_info);
 536                return -ENOMEM;
 537        }
 538
 539        atomic_inc(&txq_info->users);
 540        adap->sge.uld_txq_info[tx_uld_type] = txq_info;
 541        return 0;
 542}
 543
 544static void uld_queue_init(struct adapter *adap, unsigned int uld_type,
 545                           struct cxgb4_lld_info *lli)
 546{
 547        struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
 548
 549        lli->rxq_ids = rxq_info->rspq_id;
 550        lli->nrxq = rxq_info->nrxq;
 551        lli->ciq_ids = rxq_info->rspq_id + rxq_info->nrxq;
 552        lli->nciq = rxq_info->nciq;
 553}
 554
 555int t4_uld_mem_alloc(struct adapter *adap)
 556{
 557        struct sge *s = &adap->sge;
 558
 559        adap->uld = kcalloc(CXGB4_ULD_MAX, sizeof(*adap->uld), GFP_KERNEL);
 560        if (!adap->uld)
 561                return -ENOMEM;
 562
 563        s->uld_rxq_info = kzalloc(CXGB4_ULD_MAX *
 564                                  sizeof(struct sge_uld_rxq_info *),
 565                                  GFP_KERNEL);
 566        if (!s->uld_rxq_info)
 567                goto err_uld;
 568
 569        s->uld_txq_info = kzalloc(CXGB4_TX_MAX *
 570                                  sizeof(struct sge_uld_txq_info *),
 571                                  GFP_KERNEL);
 572        if (!s->uld_txq_info)
 573                goto err_uld_rx;
 574        return 0;
 575
 576err_uld_rx:
 577        kfree(s->uld_rxq_info);
 578err_uld:
 579        kfree(adap->uld);
 580        return -ENOMEM;
 581}
 582
 583void t4_uld_mem_free(struct adapter *adap)
 584{
 585        struct sge *s = &adap->sge;
 586
 587        kfree(s->uld_txq_info);
 588        kfree(s->uld_rxq_info);
 589        kfree(adap->uld);
 590}
 591
 592void t4_uld_clean_up(struct adapter *adap)
 593{
 594        unsigned int i;
 595
 596        if (!adap->uld)
 597                return;
 598        for (i = 0; i < CXGB4_ULD_MAX; i++) {
 599                if (!adap->uld[i].handle)
 600                        continue;
 601                if (adap->flags & FULL_INIT_DONE)
 602                        quiesce_rx_uld(adap, i);
 603                if (adap->flags & USING_MSIX)
 604                        free_msix_queue_irqs_uld(adap, i);
 605                free_sge_queues_uld(adap, i);
 606                free_queues_uld(adap, i);
 607        }
 608}
 609
 610static void uld_init(struct adapter *adap, struct cxgb4_lld_info *lld)
 611{
 612        int i;
 613
 614        lld->pdev = adap->pdev;
 615        lld->pf = adap->pf;
 616        lld->l2t = adap->l2t;
 617        lld->tids = &adap->tids;
 618        lld->ports = adap->port;
 619        lld->vr = &adap->vres;
 620        lld->mtus = adap->params.mtus;
 621        lld->ntxq = adap->sge.ofldqsets;
 622        lld->nchan = adap->params.nports;
 623        lld->nports = adap->params.nports;
 624        lld->wr_cred = adap->params.ofldq_wr_cred;
 625        lld->iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A));
 626        lld->iscsi_tagmask = t4_read_reg(adap, ULP_RX_ISCSI_TAGMASK_A);
 627        lld->iscsi_pgsz_order = t4_read_reg(adap, ULP_RX_ISCSI_PSZ_A);
 628        lld->iscsi_llimit = t4_read_reg(adap, ULP_RX_ISCSI_LLIMIT_A);
 629        lld->iscsi_ppm = &adap->iscsi_ppm;
 630        lld->adapter_type = adap->params.chip;
 631        lld->cclk_ps = 1000000000 / adap->params.vpd.cclk;
 632        lld->udb_density = 1 << adap->params.sge.eq_qpp;
 633        lld->ucq_density = 1 << adap->params.sge.iq_qpp;
 634        lld->filt_mode = adap->params.tp.vlan_pri_map;
 635        /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
 636        for (i = 0; i < NCHAN; i++)
 637                lld->tx_modq[i] = i;
 638        lld->gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A);
 639        lld->db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A);
 640        lld->fw_vers = adap->params.fw_vers;
 641        lld->dbfifo_int_thresh = dbfifo_int_thresh;
 642        lld->sge_ingpadboundary = adap->sge.fl_align;
 643        lld->sge_egrstatuspagesize = adap->sge.stat_len;
 644        lld->sge_pktshift = adap->sge.pktshift;
 645        lld->enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
 646        lld->max_ordird_qp = adap->params.max_ordird_qp;
 647        lld->max_ird_adapter = adap->params.max_ird_adapter;
 648        lld->ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
 649        lld->nodeid = dev_to_node(adap->pdev_dev);
 650        lld->fr_nsmr_tpte_wr_support = adap->params.fr_nsmr_tpte_wr_support;
 651}
 652
 653static void uld_attach(struct adapter *adap, unsigned int uld)
 654{
 655        void *handle;
 656        struct cxgb4_lld_info lli;
 657
 658        uld_init(adap, &lli);
 659        uld_queue_init(adap, uld, &lli);
 660
 661        handle = adap->uld[uld].add(&lli);
 662        if (IS_ERR(handle)) {
 663                dev_warn(adap->pdev_dev,
 664                         "could not attach to the %s driver, error %ld\n",
 665                         adap->uld[uld].name, PTR_ERR(handle));
 666                return;
 667        }
 668
 669        adap->uld[uld].handle = handle;
 670        t4_register_netevent_notifier();
 671
 672        if (adap->flags & FULL_INIT_DONE)
 673                adap->uld[uld].state_change(handle, CXGB4_STATE_UP);
 674}
 675
 676/**
 677 *      cxgb4_register_uld - register an upper-layer driver
 678 *      @type: the ULD type
 679 *      @p: the ULD methods
 680 *
 681 *      Registers an upper-layer driver with this driver and notifies the ULD
 682 *      about any presently available devices that support its type.  Returns
 683 *      %-EBUSY if a ULD of the same type is already registered.
 684 */
 685int cxgb4_register_uld(enum cxgb4_uld type,
 686                       const struct cxgb4_uld_info *p)
 687{
 688        int ret = 0;
 689        unsigned int adap_idx = 0;
 690        struct adapter *adap;
 691
 692        if (type >= CXGB4_ULD_MAX)
 693                return -EINVAL;
 694
 695        mutex_lock(&uld_mutex);
 696        list_for_each_entry(adap, &adapter_list, list_node) {
 697                if ((type == CXGB4_ULD_CRYPTO && !is_pci_uld(adap)) ||
 698                    (type != CXGB4_ULD_CRYPTO && !is_offload(adap)))
 699                        continue;
 700                if (type == CXGB4_ULD_ISCSIT && is_t4(adap->params.chip))
 701                        continue;
 702                ret = cfg_queues_uld(adap, type, p);
 703                if (ret)
 704                        goto out;
 705                ret = setup_sge_queues_uld(adap, type, p->lro);
 706                if (ret)
 707                        goto free_queues;
 708                if (adap->flags & USING_MSIX) {
 709                        name_msix_vecs_uld(adap, type);
 710                        ret = request_msix_queue_irqs_uld(adap, type);
 711                        if (ret)
 712                                goto free_rxq;
 713                }
 714                if (adap->flags & FULL_INIT_DONE)
 715                        enable_rx_uld(adap, type);
 716                if (adap->uld[type].add) {
 717                        ret = -EBUSY;
 718                        goto free_irq;
 719                }
 720                ret = setup_sge_txq_uld(adap, type, p);
 721                if (ret)
 722                        goto free_irq;
 723                adap->uld[type] = *p;
 724                uld_attach(adap, type);
 725                adap_idx++;
 726        }
 727        mutex_unlock(&uld_mutex);
 728        return 0;
 729
 730free_irq:
 731        if (adap->flags & FULL_INIT_DONE)
 732                quiesce_rx_uld(adap, type);
 733        if (adap->flags & USING_MSIX)
 734                free_msix_queue_irqs_uld(adap, type);
 735free_rxq:
 736        free_sge_queues_uld(adap, type);
 737free_queues:
 738        free_queues_uld(adap, type);
 739out:
 740
 741        list_for_each_entry(adap, &adapter_list, list_node) {
 742                if ((type == CXGB4_ULD_CRYPTO && !is_pci_uld(adap)) ||
 743                    (type != CXGB4_ULD_CRYPTO && !is_offload(adap)))
 744                        continue;
 745                if (type == CXGB4_ULD_ISCSIT && is_t4(adap->params.chip))
 746                        continue;
 747                if (!adap_idx)
 748                        break;
 749                adap->uld[type].handle = NULL;
 750                adap->uld[type].add = NULL;
 751                release_sge_txq_uld(adap, type);
 752                if (adap->flags & FULL_INIT_DONE)
 753                        quiesce_rx_uld(adap, type);
 754                if (adap->flags & USING_MSIX)
 755                        free_msix_queue_irqs_uld(adap, type);
 756                free_sge_queues_uld(adap, type);
 757                free_queues_uld(adap, type);
 758                adap_idx--;
 759        }
 760        mutex_unlock(&uld_mutex);
 761        return ret;
 762}
 763EXPORT_SYMBOL(cxgb4_register_uld);
 764
 765/**
 766 *      cxgb4_unregister_uld - unregister an upper-layer driver
 767 *      @type: the ULD type
 768 *
 769 *      Unregisters an existing upper-layer driver.
 770 */
 771int cxgb4_unregister_uld(enum cxgb4_uld type)
 772{
 773        struct adapter *adap;
 774
 775        if (type >= CXGB4_ULD_MAX)
 776                return -EINVAL;
 777
 778        mutex_lock(&uld_mutex);
 779        list_for_each_entry(adap, &adapter_list, list_node) {
 780                if ((type == CXGB4_ULD_CRYPTO && !is_pci_uld(adap)) ||
 781                    (type != CXGB4_ULD_CRYPTO && !is_offload(adap)))
 782                        continue;
 783                if (type == CXGB4_ULD_ISCSIT && is_t4(adap->params.chip))
 784                        continue;
 785                adap->uld[type].handle = NULL;
 786                adap->uld[type].add = NULL;
 787                release_sge_txq_uld(adap, type);
 788                if (adap->flags & FULL_INIT_DONE)
 789                        quiesce_rx_uld(adap, type);
 790                if (adap->flags & USING_MSIX)
 791                        free_msix_queue_irqs_uld(adap, type);
 792                free_sge_queues_uld(adap, type);
 793                free_queues_uld(adap, type);
 794        }
 795        mutex_unlock(&uld_mutex);
 796
 797        return 0;
 798}
 799EXPORT_SYMBOL(cxgb4_unregister_uld);
 800