drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (C) 2019 Chelsio Communications.  All rights reserved. */
   3
   4 #include "cxgb4.h"
   5 #include "cxgb4_tc_mqprio.h"
   6 #include "sched.h"
   7
   8 static int cxgb4_mqprio_validate(struct net_device *dev,
   9                                  struct tc_mqprio_qopt_offload *mqprio)
  10 {
  11         u64 min_rate = 0, max_rate = 0, max_link_rate;
  12         struct port_info *pi = netdev2pinfo(dev);
  13         struct adapter *adap = netdev2adap(dev);
  14         u32 qcount = 0, qoffset = 0;
  15         u32 link_ok, speed, mtu;
  16         int ret;
  17         u8 i;
  18
  19         if (!mqprio->qopt.num_tc)
  20                 return 0;
  21
  22         if (mqprio->qopt.hw != TC_MQPRIO_HW_OFFLOAD_TCS) {
  23                 netdev_err(dev, "Only full TC hardware offload is supported\n");
  24                 return -EINVAL;
  25         } else if (mqprio->mode != TC_MQPRIO_MODE_CHANNEL) {
  26                 netdev_err(dev, "Only channel mode offload is supported\n");
  27                 return -EINVAL;
  28         } else if (mqprio->shaper != TC_MQPRIO_SHAPER_BW_RATE) {
  29                 netdev_err(dev, "Only bandwidth rate shaper supported\n");
  30                 return -EINVAL;
  31         } else if (mqprio->qopt.num_tc > adap->params.nsched_cls) {
  32                 netdev_err(dev,
  33                            "Only %u traffic classes supported by hardware\n",
  34                            adap->params.nsched_cls);
  35                 return -ERANGE;
  36         }
  37
  38         ret = t4_get_link_params(pi, &link_ok, &speed, &mtu);
  39         if (ret) {
  40                 netdev_err(dev, "Failed to get link speed, ret: %d\n", ret);
  41                 return -EINVAL;
  42         }
  43
  44         /* Convert from Mbps to bps */
  45         max_link_rate = (u64)speed * 1000 * 1000;
  46
  47         for (i = 0; i < mqprio->qopt.num_tc; i++) {
  48                 qoffset = max_t(u16, mqprio->qopt.offset[i], qoffset);
  49                 qcount += mqprio->qopt.count[i];
  50
  51                 /* Convert byte per second to bits per second */
  52                 min_rate += (mqprio->min_rate[i] * 8);
  53                 max_rate += (mqprio->max_rate[i] * 8);
  54         }
  55
  56         if (qoffset >= adap->tids.neotids || qcount > adap->tids.neotids)
  57                 return -ENOMEM;
  58
  59         if (min_rate > max_link_rate || max_rate > max_link_rate) {
  60                 netdev_err(dev,
  61                            "Total Min/Max (%llu/%llu) Rate > supported (%llu)\n",
  62                            min_rate, max_rate, max_link_rate);
  63                 return -EINVAL;
  64         }
  65
  66         return 0;
  67 }
  68
  69 static int cxgb4_init_eosw_txq(struct net_device *dev,
  70                                struct sge_eosw_txq *eosw_txq,
  71                                u32 eotid, u32 hwqid)
  72 {
  73         struct adapter *adap = netdev2adap(dev);
  74         struct sge_eosw_desc *ring;
  75
  76         memset(eosw_txq, 0, sizeof(*eosw_txq));
  77
  78         ring = kcalloc(CXGB4_EOSW_TXQ_DEFAULT_DESC_NUM,
  79                        sizeof(*ring), GFP_KERNEL);
  80         if (!ring)
  81                 return -ENOMEM;
  82
  83         eosw_txq->desc = ring;
  84         eosw_txq->ndesc = CXGB4_EOSW_TXQ_DEFAULT_DESC_NUM;
  85         spin_lock_init(&eosw_txq->lock);
  86         eosw_txq->state = CXGB4_EO_STATE_CLOSED;
  87         eosw_txq->eotid = eotid;
  88         eosw_txq->hwtid = adap->tids.eotid_base + eosw_txq->eotid;
  89         eosw_txq->cred = adap->params.ofldq_wr_cred;
  90         eosw_txq->hwqid = hwqid;
  91         eosw_txq->netdev = dev;
  92         tasklet_init(&eosw_txq->qresume_tsk, cxgb4_ethofld_restart,
  93                      (unsigned long)eosw_txq);
  94         return 0;
  95 }
  96
  97 static void cxgb4_clean_eosw_txq(struct net_device *dev,
  98                                  struct sge_eosw_txq *eosw_txq)
  99 {
 100         struct adapter *adap = netdev2adap(dev);
 101
 102         cxgb4_eosw_txq_free_desc(adap, eosw_txq, eosw_txq->ndesc);
 103         eosw_txq->pidx = 0;
 104         eosw_txq->last_pidx = 0;
 105         eosw_txq->cidx = 0;
 106         eosw_txq->last_cidx = 0;
 107         eosw_txq->flowc_idx = 0;
 108         eosw_txq->inuse = 0;
 109         eosw_txq->cred = adap->params.ofldq_wr_cred;
 110         eosw_txq->ncompl = 0;
 111         eosw_txq->last_compl = 0;
 112         eosw_txq->state = CXGB4_EO_STATE_CLOSED;
 113 }
 114
 115 static void cxgb4_free_eosw_txq(struct net_device *dev,
 116                                 struct sge_eosw_txq *eosw_txq)
 117 {
 118         spin_lock_bh(&eosw_txq->lock);
 119         cxgb4_clean_eosw_txq(dev, eosw_txq);
 120         kfree(eosw_txq->desc);
 121         spin_unlock_bh(&eosw_txq->lock);
 122         tasklet_kill(&eosw_txq->qresume_tsk);
 123 }
 124
 125 static int cxgb4_mqprio_alloc_hw_resources(struct net_device *dev)
 126 {
 127         struct port_info *pi = netdev2pinfo(dev);
 128         struct adapter *adap = netdev2adap(dev);
 129         struct sge_ofld_rxq *eorxq;
 130         struct sge_eohw_txq *eotxq;
 131         int ret, msix = 0;
 132         u32 i;
 133
 134         /* Allocate ETHOFLD hardware queue structures if not done already */
 135         if (!refcount_read(&adap->tc_mqprio->refcnt)) {
 136                 adap->sge.eohw_rxq = kcalloc(adap->sge.eoqsets,
 137                                              sizeof(struct sge_ofld_rxq),
 138                                              GFP_KERNEL);
 139                 if (!adap->sge.eohw_rxq)
 140                         return -ENOMEM;
 141
 142                 adap->sge.eohw_txq = kcalloc(adap->sge.eoqsets,
 143                                              sizeof(struct sge_eohw_txq),
 144                                              GFP_KERNEL);
 145                 if (!adap->sge.eohw_txq) {
 146                         kfree(adap->sge.eohw_rxq);
 147                         return -ENOMEM;
 148                 }
 149         }
 150
 151         if (!(adap->flags & CXGB4_USING_MSIX))
 152                 msix = -((int)adap->sge.intrq.abs_id + 1);
 153
 154         for (i = 0; i < pi->nqsets; i++) {
 155                 eorxq = &adap->sge.eohw_rxq[pi->first_qset + i];
 156                 eotxq = &adap->sge.eohw_txq[pi->first_qset + i];
 157
 158                 /* Allocate Rxqs for receiving ETHOFLD Tx completions */
 159                 if (msix >= 0) {
 160                         msix = cxgb4_get_msix_idx_from_bmap(adap);
 161                         if (msix < 0)
 162                                 goto out_free_queues;
 163
 164                         eorxq->msix = &adap->msix_info[msix];
 165                         snprintf(eorxq->msix->desc,
 166                                  sizeof(eorxq->msix->desc),
 167                                  "%s-eorxq%d", dev->name, i);
 168                 }
 169
 170                 init_rspq(adap, &eorxq->rspq,
 171                           CXGB4_EOHW_RXQ_DEFAULT_INTR_USEC,
 172                           CXGB4_EOHW_RXQ_DEFAULT_PKT_CNT,
 173                           CXGB4_EOHW_RXQ_DEFAULT_DESC_NUM,
 174                           CXGB4_EOHW_RXQ_DEFAULT_DESC_SIZE);
 175
 176                 eorxq->fl.size = CXGB4_EOHW_FLQ_DEFAULT_DESC_NUM;
 177
 178                 ret = t4_sge_alloc_rxq(adap, &eorxq->rspq, false,
 179                                        dev, msix, &eorxq->fl,
 180                                        cxgb4_ethofld_rx_handler,
 181                                        NULL, 0);
 182                 if (ret)
 183                         goto out_free_queues;
 184
 185                 /* Allocate ETHOFLD hardware Txqs */
 186                 eotxq->q.size = CXGB4_EOHW_TXQ_DEFAULT_DESC_NUM;
 187                 ret = t4_sge_alloc_ethofld_txq(adap, eotxq, dev,
 188                                                eorxq->rspq.cntxt_id);
 189                 if (ret)
 190                         goto out_free_queues;
 191
 192                 /* Allocate IRQs, set IRQ affinity, and start Rx */
 193                 if (adap->flags & CXGB4_USING_MSIX) {
 194                         ret = request_irq(eorxq->msix->vec, t4_sge_intr_msix, 0,
 195                                           eorxq->msix->desc, &eorxq->rspq);
 196                         if (ret)
 197                                 goto out_free_msix;
 198
 199                         cxgb4_set_msix_aff(adap, eorxq->msix->vec,
 200                                            &eorxq->msix->aff_mask, i);
 201                 }
 202
 203                 if (adap->flags & CXGB4_FULL_INIT_DONE)
 204                         cxgb4_enable_rx(adap, &eorxq->rspq);
 205         }
 206
 207         refcount_inc(&adap->tc_mqprio->refcnt);
 208         return 0;
 209
 210 out_free_msix:
 211         while (i-- > 0) {
 212                 eorxq = &adap->sge.eohw_rxq[pi->first_qset + i];
 213
 214                 if (adap->flags & CXGB4_FULL_INIT_DONE)
 215                         cxgb4_quiesce_rx(&eorxq->rspq);
 216
 217                 if (adap->flags & CXGB4_USING_MSIX) {
 218                         cxgb4_clear_msix_aff(eorxq->msix->vec,
 219                                              eorxq->msix->aff_mask);
 220                         free_irq(eorxq->msix->vec, &eorxq->rspq);
 221                 }
 222         }
 223
 224 out_free_queues:
 225         for (i = 0; i < pi->nqsets; i++) {
 226                 eorxq = &adap->sge.eohw_rxq[pi->first_qset + i];
 227                 eotxq = &adap->sge.eohw_txq[pi->first_qset + i];
 228
 229                 if (eorxq->rspq.desc)
 230                         free_rspq_fl(adap, &eorxq->rspq, &eorxq->fl);
 231                 if (eorxq->msix)
 232                         cxgb4_free_msix_idx_in_bmap(adap, eorxq->msix->idx);
 233                 t4_sge_free_ethofld_txq(adap, eotxq);
 234         }
 235
 236         kfree(adap->sge.eohw_txq);
 237         kfree(adap->sge.eohw_rxq);
 238
 239         return ret;
 240 }
 241
 242 void cxgb4_mqprio_free_hw_resources(struct net_device *dev)
 243 {
 244         struct port_info *pi = netdev2pinfo(dev);
 245         struct adapter *adap = netdev2adap(dev);
 246         struct sge_ofld_rxq *eorxq;
 247         struct sge_eohw_txq *eotxq;
 248         u32 i;
 249
 250         /* Return if no ETHOFLD structures have been allocated yet */
 251         if (!refcount_read(&adap->tc_mqprio->refcnt))
 252                 return;
 253
 254         /* Return if no hardware queues have been allocated */
 255         if (!adap->sge.eohw_rxq[pi->first_qset].rspq.desc)
 256                 return;
 257
 258         for (i = 0; i < pi->nqsets; i++) {
 259                 eorxq = &adap->sge.eohw_rxq[pi->first_qset + i];
 260                 eotxq = &adap->sge.eohw_txq[pi->first_qset + i];
 261
 262                 /* Device removal path will already disable NAPI
 263                  * before unregistering netdevice. So, only disable
 264                  * NAPI if we're not in device removal path
 265                  */
 266                 if (!(adap->flags & CXGB4_SHUTTING_DOWN))
 267                         cxgb4_quiesce_rx(&eorxq->rspq);
 268
 269                 if (adap->flags & CXGB4_USING_MSIX) {
 270                         cxgb4_clear_msix_aff(eorxq->msix->vec,
 271                                              eorxq->msix->aff_mask);
 272                         free_irq(eorxq->msix->vec, &eorxq->rspq);
 273                 }
 274
 275                 free_rspq_fl(adap, &eorxq->rspq, &eorxq->fl);
 276                 t4_sge_free_ethofld_txq(adap, eotxq);
 277         }
 278
 279         /* Free up ETHOFLD structures if there are no users */
 280         if (refcount_dec_and_test(&adap->tc_mqprio->refcnt)) {
 281                 kfree(adap->sge.eohw_txq);
 282                 kfree(adap->sge.eohw_rxq);
 283         }
 284 }
 285
 286 static int cxgb4_mqprio_alloc_tc(struct net_device *dev,
 287                                  struct tc_mqprio_qopt_offload *mqprio)
 288 {
 289         struct ch_sched_params p = {
 290                 .type = SCHED_CLASS_TYPE_PACKET,
 291                 .u.params.level = SCHED_CLASS_LEVEL_CL_RL,
 292                 .u.params.mode = SCHED_CLASS_MODE_FLOW,
 293                 .u.params.rateunit = SCHED_CLASS_RATEUNIT_BITS,
 294                 .u.params.ratemode = SCHED_CLASS_RATEMODE_ABS,
 295                 .u.params.class = SCHED_CLS_NONE,
 296                 .u.params.weight = 0,
 297                 .u.params.pktsize = dev->mtu,
 298         };
 299         struct cxgb4_tc_port_mqprio *tc_port_mqprio;
 300         struct port_info *pi = netdev2pinfo(dev);
 301         struct adapter *adap = netdev2adap(dev);
 302         struct sched_class *e;
 303         int ret;
 304         u8 i;
 305
 306         tc_port_mqprio = &adap->tc_mqprio->port_mqprio[pi->port_id];
 307         p.u.params.channel = pi->tx_chan;
 308         for (i = 0; i < mqprio->qopt.num_tc; i++) {
 309                 /* Convert from bytes per second to Kbps */
 310                 p.u.params.minrate = mqprio->min_rate[i] * 8 / 1000;
 311                 p.u.params.maxrate = mqprio->max_rate[i] * 8 / 1000;
 312
 313                 e = cxgb4_sched_class_alloc(dev, &p);
 314                 if (!e) {
 315                         ret = -ENOMEM;
 316                         goto out_err;
 317                 }
 318
 319                 tc_port_mqprio->tc_hwtc_map[i] = e->idx;
 320         }
 321
 322         return 0;
 323
 324 out_err:
 325         while (i--)
 326                 cxgb4_sched_class_free(dev, tc_port_mqprio->tc_hwtc_map[i]);
 327
 328         return ret;
 329 }
 330
 331 static void cxgb4_mqprio_free_tc(struct net_device *dev)
 332 {
 333         struct cxgb4_tc_port_mqprio *tc_port_mqprio;
 334         struct port_info *pi = netdev2pinfo(dev);
 335         struct adapter *adap = netdev2adap(dev);
 336         u8 i;
 337
 338         tc_port_mqprio = &adap->tc_mqprio->port_mqprio[pi->port_id];
 339         for (i = 0; i < tc_port_mqprio->mqprio.qopt.num_tc; i++)
 340                 cxgb4_sched_class_free(dev, tc_port_mqprio->tc_hwtc_map[i]);
 341 }
 342
 343 static int cxgb4_mqprio_class_bind(struct net_device *dev,
 344                                    struct sge_eosw_txq *eosw_txq,
 345                                    u8 tc)
 346 {
 347         struct ch_sched_flowc fe;
 348         int ret;
 349
 350         init_completion(&eosw_txq->completion);
 351
 352         fe.tid = eosw_txq->eotid;
 353         fe.class = tc;
 354
 355         ret = cxgb4_sched_class_bind(dev, &fe, SCHED_FLOWC);
 356         if (ret)
 357                 return ret;
 358
 359         ret = wait_for_completion_timeout(&eosw_txq->completion,
 360                                           CXGB4_FLOWC_WAIT_TIMEOUT);
 361         if (!ret)
 362                 return -ETIMEDOUT;
 363
 364         return 0;
 365 }
 366
 367 static void cxgb4_mqprio_class_unbind(struct net_device *dev,
 368                                       struct sge_eosw_txq *eosw_txq,
 369                                       u8 tc)
 370 {
 371         struct adapter *adap = netdev2adap(dev);
 372         struct ch_sched_flowc fe;
 373
 374         /* If we're shutting down, interrupts are disabled and no completions
 375          * come back. So, skip waiting for completions in this scenario.
 376          */
 377         if (!(adap->flags & CXGB4_SHUTTING_DOWN))
 378                 init_completion(&eosw_txq->completion);
 379
 380         fe.tid = eosw_txq->eotid;
 381         fe.class = tc;
 382         cxgb4_sched_class_unbind(dev, &fe, SCHED_FLOWC);
 383
 384         if (!(adap->flags & CXGB4_SHUTTING_DOWN))
 385                 wait_for_completion_timeout(&eosw_txq->completion,
 386                                             CXGB4_FLOWC_WAIT_TIMEOUT);
 387 }
 388
 389 static int cxgb4_mqprio_enable_offload(struct net_device *dev,
 390                                        struct tc_mqprio_qopt_offload *mqprio)
 391 {
 392         struct cxgb4_tc_port_mqprio *tc_port_mqprio;
 393         u32 qoffset, qcount, tot_qcount, qid, hwqid;
 394         struct port_info *pi = netdev2pinfo(dev);
 395         struct adapter *adap = netdev2adap(dev);
 396         struct sge_eosw_txq *eosw_txq;
 397         int eotid, ret;
 398         u16 i, j;
 399         u8 hwtc;
 400
 401         ret = cxgb4_mqprio_alloc_hw_resources(dev);
 402         if (ret)
 403                 return -ENOMEM;
 404
 405         tc_port_mqprio = &adap->tc_mqprio->port_mqprio[pi->port_id];
 406         for (i = 0; i < mqprio->qopt.num_tc; i++) {
 407                 qoffset = mqprio->qopt.offset[i];
 408                 qcount = mqprio->qopt.count[i];
 409                 for (j = 0; j < qcount; j++) {
 410                         eotid = cxgb4_get_free_eotid(&adap->tids);
 411                         if (eotid < 0) {
 412                                 ret = -ENOMEM;
 413                                 goto out_free_eotids;
 414                         }
 415
 416                         qid = qoffset + j;
 417                         hwqid = pi->first_qset + (eotid % pi->nqsets);
 418                         eosw_txq = &tc_port_mqprio->eosw_txq[qid];
 419                         ret = cxgb4_init_eosw_txq(dev, eosw_txq,
 420                                                   eotid, hwqid);
 421                         if (ret)
 422                                 goto out_free_eotids;
 423
 424                         cxgb4_alloc_eotid(&adap->tids, eotid, eosw_txq);
 425
 426                         hwtc = tc_port_mqprio->tc_hwtc_map[i];
 427                         ret = cxgb4_mqprio_class_bind(dev, eosw_txq, hwtc);
 428                         if (ret)
 429                                 goto out_free_eotids;
 430                 }
 431         }
 432
 433         memcpy(&tc_port_mqprio->mqprio, mqprio,
 434                sizeof(struct tc_mqprio_qopt_offload));
 435
 436         /* Inform the stack about the configured tc params.
 437          *
 438          * Set the correct queue map. If no queue count has been
 439          * specified, then send the traffic through default NIC
 440          * queues; instead of ETHOFLD queues.
 441          */
 442         ret = netdev_set_num_tc(dev, mqprio->qopt.num_tc);
 443         if (ret)
 444                 goto out_free_eotids;
 445
 446         tot_qcount = pi->nqsets;
 447         for (i = 0; i < mqprio->qopt.num_tc; i++) {
 448                 qcount = mqprio->qopt.count[i];
 449                 if (qcount) {
 450                         qoffset = mqprio->qopt.offset[i] + pi->nqsets;
 451                 } else {
 452                         qcount = pi->nqsets;
 453                         qoffset = 0;
 454                 }
 455
 456                 ret = netdev_set_tc_queue(dev, i, qcount, qoffset);
 457                 if (ret)
 458                         goto out_reset_tc;
 459
 460                 tot_qcount += mqprio->qopt.count[i];
 461         }
 462
 463         ret = netif_set_real_num_tx_queues(dev, tot_qcount);
 464         if (ret)
 465                 goto out_reset_tc;
 466
 467         tc_port_mqprio->state = CXGB4_MQPRIO_STATE_ACTIVE;
 468         return 0;
 469
 470 out_reset_tc:
 471         netdev_reset_tc(dev);
 472         i = mqprio->qopt.num_tc;
 473
 474 out_free_eotids:
 475         while (i-- > 0) {
 476                 qoffset = mqprio->qopt.offset[i];
 477                 qcount = mqprio->qopt.count[i];
 478                 for (j = 0; j < qcount; j++) {
 479                         eosw_txq = &tc_port_mqprio->eosw_txq[qoffset + j];
 480
 481                         hwtc = tc_port_mqprio->tc_hwtc_map[i];
 482                         cxgb4_mqprio_class_unbind(dev, eosw_txq, hwtc);
 483
 484                         cxgb4_free_eotid(&adap->tids, eosw_txq->eotid);
 485                         cxgb4_free_eosw_txq(dev, eosw_txq);
 486                 }
 487         }
 488
 489         cxgb4_mqprio_free_hw_resources(dev);
 490         return ret;
 491 }
 492
 493 static void cxgb4_mqprio_disable_offload(struct net_device *dev)
 494 {
 495         struct cxgb4_tc_port_mqprio *tc_port_mqprio;
 496         struct port_info *pi = netdev2pinfo(dev);
 497         struct adapter *adap = netdev2adap(dev);
 498         struct sge_eosw_txq *eosw_txq;
 499         u32 qoffset, qcount;
 500         u16 i, j;
 501         u8 hwtc;
 502
 503         tc_port_mqprio = &adap->tc_mqprio->port_mqprio[pi->port_id];
 504         if (tc_port_mqprio->state != CXGB4_MQPRIO_STATE_ACTIVE)
 505                 return;
 506
 507         netdev_reset_tc(dev);
 508         netif_set_real_num_tx_queues(dev, pi->nqsets);
 509
 510         for (i = 0; i < tc_port_mqprio->mqprio.qopt.num_tc; i++) {
 511                 qoffset = tc_port_mqprio->mqprio.qopt.offset[i];
 512                 qcount = tc_port_mqprio->mqprio.qopt.count[i];
 513                 for (j = 0; j < qcount; j++) {
 514                         eosw_txq = &tc_port_mqprio->eosw_txq[qoffset + j];
 515
 516                         hwtc = tc_port_mqprio->tc_hwtc_map[i];
 517                         cxgb4_mqprio_class_unbind(dev, eosw_txq, hwtc);
 518
 519                         cxgb4_free_eotid(&adap->tids, eosw_txq->eotid);
 520                         cxgb4_free_eosw_txq(dev, eosw_txq);
 521                 }
 522         }
 523
 524         cxgb4_mqprio_free_hw_resources(dev);
 525
 526         /* Free up the traffic classes */
 527         cxgb4_mqprio_free_tc(dev);
 528
 529         memset(&tc_port_mqprio->mqprio, 0,
 530                sizeof(struct tc_mqprio_qopt_offload));
 531
 532         tc_port_mqprio->state = CXGB4_MQPRIO_STATE_DISABLED;
 533 }
 534
 535 int cxgb4_setup_tc_mqprio(struct net_device *dev,
 536                           struct tc_mqprio_qopt_offload *mqprio)
 537 {
 538         bool needs_bring_up = false;
 539         int ret;
 540
 541         ret = cxgb4_mqprio_validate(dev, mqprio);
 542         if (ret)
 543                 return ret;
 544
 545         /* To configure tc params, the current allocated EOTIDs must
 546          * be freed up. However, they can't be freed up if there's
 547          * traffic running on the interface. So, ensure interface is
 548          * down before configuring tc params.
 549          */
 550         if (netif_running(dev)) {
 551                 cxgb_close(dev);
 552                 needs_bring_up = true;
 553         }
 554
 555         cxgb4_mqprio_disable_offload(dev);
 556
 557         /* If requested for clear, then just return since resources are
 558          * already freed up by now.
 559          */
 560         if (!mqprio->qopt.num_tc)
 561                 goto out;
 562
 563         /* Allocate free available traffic classes and configure
 564          * their rate parameters.
 565          */
 566         ret = cxgb4_mqprio_alloc_tc(dev, mqprio);
 567         if (ret)
 568                 goto out;
 569
 570         ret = cxgb4_mqprio_enable_offload(dev, mqprio);
 571         if (ret) {
 572                 cxgb4_mqprio_free_tc(dev);
 573                 goto out;
 574         }
 575
 576 out:
 577         if (needs_bring_up)
 578                 cxgb_open(dev);
 579
 580         return ret;
 581 }
 582
 583 int cxgb4_init_tc_mqprio(struct adapter *adap)
 584 {
 585         struct cxgb4_tc_port_mqprio *tc_port_mqprio, *port_mqprio;
 586         struct cxgb4_tc_mqprio *tc_mqprio;
 587         struct sge_eosw_txq *eosw_txq;
 588         int ret = 0;
 589         u8 i;
 590
 591         tc_mqprio = kzalloc(sizeof(*tc_mqprio), GFP_KERNEL);
 592         if (!tc_mqprio)
 593                 return -ENOMEM;
 594
 595         tc_port_mqprio = kcalloc(adap->params.nports, sizeof(*tc_port_mqprio),
 596                                  GFP_KERNEL);
 597         if (!tc_port_mqprio) {
 598                 ret = -ENOMEM;
 599                 goto out_free_mqprio;
 600         }
 601
 602         tc_mqprio->port_mqprio = tc_port_mqprio;
 603         for (i = 0; i < adap->params.nports; i++) {
 604                 port_mqprio = &tc_mqprio->port_mqprio[i];
 605                 eosw_txq = kcalloc(adap->tids.neotids, sizeof(*eosw_txq),
 606                                    GFP_KERNEL);
 607                 if (!eosw_txq) {
 608                         ret = -ENOMEM;
 609                         goto out_free_ports;
 610                 }
 611                 port_mqprio->eosw_txq = eosw_txq;
 612         }
 613
 614         adap->tc_mqprio = tc_mqprio;
 615         refcount_set(&adap->tc_mqprio->refcnt, 0);
 616         return 0;
 617
 618 out_free_ports:
 619         for (i = 0; i < adap->params.nports; i++) {
 620                 port_mqprio = &tc_mqprio->port_mqprio[i];
 621                 kfree(port_mqprio->eosw_txq);
 622         }
 623         kfree(tc_port_mqprio);
 624
 625 out_free_mqprio:
 626         kfree(tc_mqprio);
 627         return ret;
 628 }
 629
 630 void cxgb4_cleanup_tc_mqprio(struct adapter *adap)
 631 {
 632         struct cxgb4_tc_port_mqprio *port_mqprio;
 633         u8 i;
 634
 635         if (adap->tc_mqprio) {
 636                 if (adap->tc_mqprio->port_mqprio) {
 637                         for (i = 0; i < adap->params.nports; i++) {
 638                                 struct net_device *dev = adap->port[i];
 639
 640                                 if (dev)
 641                                         cxgb4_mqprio_disable_offload(dev);
 642                                 port_mqprio = &adap->tc_mqprio->port_mqprio[i];
 643                                 kfree(port_mqprio->eosw_txq);
 644                         }
 645                         kfree(adap->tc_mqprio->port_mqprio);
 646                 }
 647                 kfree(adap->tc_mqprio);
 648         }
 649 }