drivers/dma/tegra210-adma.c

   1 /*
   2  * ADMA driver for Nvidia's Tegra210 ADMA controller.
   3  *
   4  * Copyright (c) 2016, NVIDIA CORPORATION.  All rights reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/clk.h>
  20 #include <linux/iopoll.h>
  21 #include <linux/module.h>
  22 #include <linux/of_device.h>
  23 #include <linux/of_dma.h>
  24 #include <linux/of_irq.h>
  25 #include <linux/pm_runtime.h>
  26 #include <linux/slab.h>
  27
  28 #include "virt-dma.h"
  29
  30 #define ADMA_CH_CMD                                     0x00
  31 #define ADMA_CH_STATUS                                  0x0c
  32 #define ADMA_CH_STATUS_XFER_EN                          BIT(0)
  33 #define ADMA_CH_STATUS_XFER_PAUSED                      BIT(1)
  34
  35 #define ADMA_CH_INT_STATUS                              0x10
  36 #define ADMA_CH_INT_STATUS_XFER_DONE                    BIT(0)
  37
  38 #define ADMA_CH_INT_CLEAR                               0x1c
  39 #define ADMA_CH_CTRL                                    0x24
  40 #define ADMA_CH_CTRL_DIR(val)                           (((val) & 0xf) << 12)
  41 #define ADMA_CH_CTRL_DIR_AHUB2MEM                       2
  42 #define ADMA_CH_CTRL_DIR_MEM2AHUB                       4
  43 #define ADMA_CH_CTRL_MODE_CONTINUOUS                    (2 << 8)
  44 #define ADMA_CH_CTRL_FLOWCTRL_EN                        BIT(1)
  45 #define ADMA_CH_CTRL_XFER_PAUSE_SHIFT                   0
  46
  47 #define ADMA_CH_CONFIG                                  0x28
  48 #define ADMA_CH_CONFIG_SRC_BUF(val)                     (((val) & 0x7) << 28)
  49 #define ADMA_CH_CONFIG_TRG_BUF(val)                     (((val) & 0x7) << 24)
  50 #define ADMA_CH_CONFIG_BURST_SIZE_SHIFT                 20
  51 #define ADMA_CH_CONFIG_MAX_BURST_SIZE                   16
  52 #define ADMA_CH_CONFIG_WEIGHT_FOR_WRR(val)              ((val) & 0xf)
  53 #define ADMA_CH_CONFIG_MAX_BUFS                         8
  54
  55 #define ADMA_CH_FIFO_CTRL                               0x2c
  56 #define TEGRA210_ADMA_CH_FIFO_CTRL_OFLWTHRES(val)       (((val) & 0xf) << 24)
  57 #define TEGRA210_ADMA_CH_FIFO_CTRL_STRVTHRES(val)       (((val) & 0xf) << 16)
  58 #define TEGRA210_ADMA_CH_FIFO_CTRL_TXSIZE(val)          (((val) & 0xf) << 8)
  59 #define TEGRA210_ADMA_CH_FIFO_CTRL_RXSIZE(val)          ((val) & 0xf)
  60 #define TEGRA186_ADMA_CH_FIFO_CTRL_OFLWTHRES(val)       (((val) & 0x1f) << 24)
  61 #define TEGRA186_ADMA_CH_FIFO_CTRL_STRVTHRES(val)       (((val) & 0x1f) << 16)
  62 #define TEGRA186_ADMA_CH_FIFO_CTRL_TXSIZE(val)          (((val) & 0x1f) << 8)
  63 #define TEGRA186_ADMA_CH_FIFO_CTRL_RXSIZE(val)          ((val) & 0x1f)
  64
  65 #define ADMA_CH_LOWER_SRC_ADDR                          0x34
  66 #define ADMA_CH_LOWER_TRG_ADDR                          0x3c
  67 #define ADMA_CH_TC                                      0x44
  68 #define ADMA_CH_TC_COUNT_MASK                           0x3ffffffc
  69
  70 #define ADMA_CH_XFER_STATUS                             0x54
  71 #define ADMA_CH_XFER_STATUS_COUNT_MASK                  0xffff
  72
  73 #define ADMA_GLOBAL_CMD                                 0x00
  74 #define ADMA_GLOBAL_SOFT_RESET                          0x04
  75
  76 #define TEGRA_ADMA_BURST_COMPLETE_TIME                  20
  77
  78 #define TEGRA210_FIFO_CTRL_DEFAULT (TEGRA210_ADMA_CH_FIFO_CTRL_OFLWTHRES(1) | \
  79                                     TEGRA210_ADMA_CH_FIFO_CTRL_STRVTHRES(1) | \
  80                                     TEGRA210_ADMA_CH_FIFO_CTRL_TXSIZE(3)    | \
  81                                     TEGRA210_ADMA_CH_FIFO_CTRL_RXSIZE(3))
  82
  83 #define TEGRA186_FIFO_CTRL_DEFAULT (TEGRA186_ADMA_CH_FIFO_CTRL_OFLWTHRES(1) | \
  84                                     TEGRA186_ADMA_CH_FIFO_CTRL_STRVTHRES(1) | \
  85                                     TEGRA186_ADMA_CH_FIFO_CTRL_TXSIZE(3)    | \
  86                                     TEGRA186_ADMA_CH_FIFO_CTRL_RXSIZE(3))
  87
  88 #define ADMA_CH_REG_FIELD_VAL(val, mask, shift) (((val) & mask) << shift)
  89
  90 struct tegra_adma;
  91
  92 /*
  93  * struct tegra_adma_chip_data - Tegra chip specific data
  94  * @global_reg_offset: Register offset of DMA global register.
  95  * @global_int_clear: Register offset of DMA global interrupt clear.
  96  * @ch_req_tx_shift: Register offset for AHUB transmit channel select.
  97  * @ch_req_rx_shift: Register offset for AHUB receive channel select.
  98  * @ch_base_offset: Register offset of DMA channel registers.
  99  * @ch_fifo_ctrl: Default value for channel FIFO CTRL register.
 100  * @ch_req_mask: Mask for Tx or Rx channel select.
 101  * @ch_req_max: Maximum number of Tx or Rx channels available.
 102  * @ch_reg_size: Size of DMA channel register space.
 103  * @nr_channels: Number of DMA channels available.
 104  */
 105 struct tegra_adma_chip_data {
 106         unsigned int (*adma_get_burst_config)(unsigned int burst_size);
 107         unsigned int global_reg_offset;
 108         unsigned int global_int_clear;
 109         unsigned int ch_req_tx_shift;
 110         unsigned int ch_req_rx_shift;
 111         unsigned int ch_base_offset;
 112         unsigned int ch_fifo_ctrl;
 113         unsigned int ch_req_mask;
 114         unsigned int ch_req_max;
 115         unsigned int ch_reg_size;
 116         unsigned int nr_channels;
 117 };
 118
 119 /*
 120  * struct tegra_adma_chan_regs - Tegra ADMA channel registers
 121  */
 122 struct tegra_adma_chan_regs {
 123         unsigned int ctrl;
 124         unsigned int config;
 125         unsigned int src_addr;
 126         unsigned int trg_addr;
 127         unsigned int fifo_ctrl;
 128         unsigned int cmd;
 129         unsigned int tc;
 130 };
 131
 132 /*
 133  * struct tegra_adma_desc - Tegra ADMA descriptor to manage transfer requests.
 134  */
 135 struct tegra_adma_desc {
 136         struct virt_dma_desc            vd;
 137         struct tegra_adma_chan_regs     ch_regs;
 138         size_t                          buf_len;
 139         size_t                          period_len;
 140         size_t                          num_periods;
 141 };
 142
 143 /*
 144  * struct tegra_adma_chan - Tegra ADMA channel information
 145  */
 146 struct tegra_adma_chan {
 147         struct virt_dma_chan            vc;
 148         struct tegra_adma_desc          *desc;
 149         struct tegra_adma               *tdma;
 150         int                             irq;
 151         void __iomem                    *chan_addr;
 152
 153         /* Slave channel configuration info */
 154         struct dma_slave_config         sconfig;
 155         enum dma_transfer_direction     sreq_dir;
 156         unsigned int                    sreq_index;
 157         bool                            sreq_reserved;
 158         struct tegra_adma_chan_regs     ch_regs;
 159
 160         /* Transfer count and position info */
 161         unsigned int                    tx_buf_count;
 162         unsigned int                    tx_buf_pos;
 163 };
 164
 165 /*
 166  * struct tegra_adma - Tegra ADMA controller information
 167  */
 168 struct tegra_adma {
 169         struct dma_device               dma_dev;
 170         struct device                   *dev;
 171         void __iomem                    *base_addr;
 172         struct clk                      *ahub_clk;
 173         unsigned int                    nr_channels;
 174         unsigned long                   rx_requests_reserved;
 175         unsigned long                   tx_requests_reserved;
 176
 177         /* Used to store global command register state when suspending */
 178         unsigned int                    global_cmd;
 179
 180         const struct tegra_adma_chip_data *cdata;
 181
 182         /* Last member of the structure */
 183         struct tegra_adma_chan          channels[0];
 184 };
 185
 186 static inline void tdma_write(struct tegra_adma *tdma, u32 reg, u32 val)
 187 {
 188         writel(val, tdma->base_addr + tdma->cdata->global_reg_offset + reg);
 189 }
 190
 191 static inline u32 tdma_read(struct tegra_adma *tdma, u32 reg)
 192 {
 193         return readl(tdma->base_addr + tdma->cdata->global_reg_offset + reg);
 194 }
 195
 196 static inline void tdma_ch_write(struct tegra_adma_chan *tdc, u32 reg, u32 val)
 197 {
 198         writel(val, tdc->chan_addr + reg);
 199 }
 200
 201 static inline u32 tdma_ch_read(struct tegra_adma_chan *tdc, u32 reg)
 202 {
 203         return readl(tdc->chan_addr + reg);
 204 }
 205
 206 static inline struct tegra_adma_chan *to_tegra_adma_chan(struct dma_chan *dc)
 207 {
 208         return container_of(dc, struct tegra_adma_chan, vc.chan);
 209 }
 210
 211 static inline struct tegra_adma_desc *to_tegra_adma_desc(
 212                 struct dma_async_tx_descriptor *td)
 213 {
 214         return container_of(td, struct tegra_adma_desc, vd.tx);
 215 }
 216
 217 static inline struct device *tdc2dev(struct tegra_adma_chan *tdc)
 218 {
 219         return tdc->tdma->dev;
 220 }
 221
 222 static void tegra_adma_desc_free(struct virt_dma_desc *vd)
 223 {
 224         kfree(container_of(vd, struct tegra_adma_desc, vd));
 225 }
 226
 227 static int tegra_adma_slave_config(struct dma_chan *dc,
 228                                    struct dma_slave_config *sconfig)
 229 {
 230         struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
 231
 232         memcpy(&tdc->sconfig, sconfig, sizeof(*sconfig));
 233
 234         return 0;
 235 }
 236
 237 static int tegra_adma_init(struct tegra_adma *tdma)
 238 {
 239         u32 status;
 240         int ret;
 241
 242         /* Clear any interrupts */
 243         tdma_write(tdma, tdma->cdata->global_int_clear, 0x1);
 244
 245         /* Assert soft reset */
 246         tdma_write(tdma, ADMA_GLOBAL_SOFT_RESET, 0x1);
 247
 248         /* Wait for reset to clear */
 249         ret = readx_poll_timeout(readl,
 250                                  tdma->base_addr +
 251                                  tdma->cdata->global_reg_offset +
 252                                  ADMA_GLOBAL_SOFT_RESET,
 253                                  status, status == 0, 20, 10000);
 254         if (ret)
 255                 return ret;
 256
 257         /* Enable global ADMA registers */
 258         tdma_write(tdma, ADMA_GLOBAL_CMD, 1);
 259
 260         return 0;
 261 }
 262
 263 static int tegra_adma_request_alloc(struct tegra_adma_chan *tdc,
 264                                     enum dma_transfer_direction direction)
 265 {
 266         struct tegra_adma *tdma = tdc->tdma;
 267         unsigned int sreq_index = tdc->sreq_index;
 268
 269         if (tdc->sreq_reserved)
 270                 return tdc->sreq_dir == direction ? 0 : -EINVAL;
 271
 272         if (sreq_index > tdma->cdata->ch_req_max) {
 273                 dev_err(tdma->dev, "invalid DMA request\n");
 274                 return -EINVAL;
 275         }
 276
 277         switch (direction) {
 278         case DMA_MEM_TO_DEV:
 279                 if (test_and_set_bit(sreq_index, &tdma->tx_requests_reserved)) {
 280                         dev_err(tdma->dev, "DMA request reserved\n");
 281                         return -EINVAL;
 282                 }
 283                 break;
 284
 285         case DMA_DEV_TO_MEM:
 286                 if (test_and_set_bit(sreq_index, &tdma->rx_requests_reserved)) {
 287                         dev_err(tdma->dev, "DMA request reserved\n");
 288                         return -EINVAL;
 289                 }
 290                 break;
 291
 292         default:
 293                 dev_WARN(tdma->dev, "channel %s has invalid transfer type\n",
 294                          dma_chan_name(&tdc->vc.chan));
 295                 return -EINVAL;
 296         }
 297
 298         tdc->sreq_dir = direction;
 299         tdc->sreq_reserved = true;
 300
 301         return 0;
 302 }
 303
 304 static void tegra_adma_request_free(struct tegra_adma_chan *tdc)
 305 {
 306         struct tegra_adma *tdma = tdc->tdma;
 307
 308         if (!tdc->sreq_reserved)
 309                 return;
 310
 311         switch (tdc->sreq_dir) {
 312         case DMA_MEM_TO_DEV:
 313                 clear_bit(tdc->sreq_index, &tdma->tx_requests_reserved);
 314                 break;
 315
 316         case DMA_DEV_TO_MEM:
 317                 clear_bit(tdc->sreq_index, &tdma->rx_requests_reserved);
 318                 break;
 319
 320         default:
 321                 dev_WARN(tdma->dev, "channel %s has invalid transfer type\n",
 322                          dma_chan_name(&tdc->vc.chan));
 323                 return;
 324         }
 325
 326         tdc->sreq_reserved = false;
 327 }
 328
 329 static u32 tegra_adma_irq_status(struct tegra_adma_chan *tdc)
 330 {
 331         u32 status = tdma_ch_read(tdc, ADMA_CH_INT_STATUS);
 332
 333         return status & ADMA_CH_INT_STATUS_XFER_DONE;
 334 }
 335
 336 static u32 tegra_adma_irq_clear(struct tegra_adma_chan *tdc)
 337 {
 338         u32 status = tegra_adma_irq_status(tdc);
 339
 340         if (status)
 341                 tdma_ch_write(tdc, ADMA_CH_INT_CLEAR, status);
 342
 343         return status;
 344 }
 345
 346 static void tegra_adma_stop(struct tegra_adma_chan *tdc)
 347 {
 348         unsigned int status;
 349
 350         /* Disable ADMA */
 351         tdma_ch_write(tdc, ADMA_CH_CMD, 0);
 352
 353         /* Clear interrupt status */
 354         tegra_adma_irq_clear(tdc);
 355
 356         if (readx_poll_timeout_atomic(readl, tdc->chan_addr + ADMA_CH_STATUS,
 357                         status, !(status & ADMA_CH_STATUS_XFER_EN),
 358                         20, 10000)) {
 359                 dev_err(tdc2dev(tdc), "unable to stop DMA channel\n");
 360                 return;
 361         }
 362
 363         kfree(tdc->desc);
 364         tdc->desc = NULL;
 365 }
 366
 367 static void tegra_adma_start(struct tegra_adma_chan *tdc)
 368 {
 369         struct virt_dma_desc *vd = vchan_next_desc(&tdc->vc);
 370         struct tegra_adma_chan_regs *ch_regs;
 371         struct tegra_adma_desc *desc;
 372
 373         if (!vd)
 374                 return;
 375
 376         list_del(&vd->node);
 377
 378         desc = to_tegra_adma_desc(&vd->tx);
 379
 380         if (!desc) {
 381                 dev_warn(tdc2dev(tdc), "unable to start DMA, no descriptor\n");
 382                 return;
 383         }
 384
 385         ch_regs = &desc->ch_regs;
 386
 387         tdc->tx_buf_pos = 0;
 388         tdc->tx_buf_count = 0;
 389         tdma_ch_write(tdc, ADMA_CH_TC, ch_regs->tc);
 390         tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);
 391         tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, ch_regs->src_addr);
 392         tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, ch_regs->trg_addr);
 393         tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_regs->fifo_ctrl);
 394         tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_regs->config);
 395
 396         /* Start ADMA */
 397         tdma_ch_write(tdc, ADMA_CH_CMD, 1);
 398
 399         tdc->desc = desc;
 400 }
 401
 402 static unsigned int tegra_adma_get_residue(struct tegra_adma_chan *tdc)
 403 {
 404         struct tegra_adma_desc *desc = tdc->desc;
 405         unsigned int max = ADMA_CH_XFER_STATUS_COUNT_MASK + 1;
 406         unsigned int pos = tdma_ch_read(tdc, ADMA_CH_XFER_STATUS);
 407         unsigned int periods_remaining;
 408
 409         /*
 410          * Handle wrap around of buffer count register
 411          */
 412         if (pos < tdc->tx_buf_pos)
 413                 tdc->tx_buf_count += pos + (max - tdc->tx_buf_pos);
 414         else
 415                 tdc->tx_buf_count += pos - tdc->tx_buf_pos;
 416
 417         periods_remaining = tdc->tx_buf_count % desc->num_periods;
 418         tdc->tx_buf_pos = pos;
 419
 420         return desc->buf_len - (periods_remaining * desc->period_len);
 421 }
 422
 423 static irqreturn_t tegra_adma_isr(int irq, void *dev_id)
 424 {
 425         struct tegra_adma_chan *tdc = dev_id;
 426         unsigned long status;
 427         unsigned long flags;
 428
 429         spin_lock_irqsave(&tdc->vc.lock, flags);
 430
 431         status = tegra_adma_irq_clear(tdc);
 432         if (status == 0 || !tdc->desc) {
 433                 spin_unlock_irqrestore(&tdc->vc.lock, flags);
 434                 return IRQ_NONE;
 435         }
 436
 437         vchan_cyclic_callback(&tdc->desc->vd);
 438
 439         spin_unlock_irqrestore(&tdc->vc.lock, flags);
 440
 441         return IRQ_HANDLED;
 442 }
 443
 444 static void tegra_adma_issue_pending(struct dma_chan *dc)
 445 {
 446         struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
 447         unsigned long flags;
 448
 449         spin_lock_irqsave(&tdc->vc.lock, flags);
 450
 451         if (vchan_issue_pending(&tdc->vc)) {
 452                 if (!tdc->desc)
 453                         tegra_adma_start(tdc);
 454         }
 455
 456         spin_unlock_irqrestore(&tdc->vc.lock, flags);
 457 }
 458
 459 static bool tegra_adma_is_paused(struct tegra_adma_chan *tdc)
 460 {
 461         u32 csts;
 462
 463         csts = tdma_ch_read(tdc, ADMA_CH_STATUS);
 464         csts &= ADMA_CH_STATUS_XFER_PAUSED;
 465
 466         return csts ? true : false;
 467 }
 468
 469 static int tegra_adma_pause(struct dma_chan *dc)
 470 {
 471         struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
 472         struct tegra_adma_desc *desc = tdc->desc;
 473         struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;
 474         int dcnt = 10;
 475
 476         ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);
 477         ch_regs->ctrl |= (1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT);
 478         tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);
 479
 480         while (dcnt-- && !tegra_adma_is_paused(tdc))
 481                 udelay(TEGRA_ADMA_BURST_COMPLETE_TIME);
 482
 483         if (dcnt < 0) {
 484                 dev_err(tdc2dev(tdc), "unable to pause DMA channel\n");
 485                 return -EBUSY;
 486         }
 487
 488         return 0;
 489 }
 490
 491 static int tegra_adma_resume(struct dma_chan *dc)
 492 {
 493         struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
 494         struct tegra_adma_desc *desc = tdc->desc;
 495         struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;
 496
 497         ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);
 498         ch_regs->ctrl &= ~(1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT);
 499         tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);
 500
 501         return 0;
 502 }
 503
 504 static int tegra_adma_terminate_all(struct dma_chan *dc)
 505 {
 506         struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
 507         unsigned long flags;
 508         LIST_HEAD(head);
 509
 510         spin_lock_irqsave(&tdc->vc.lock, flags);
 511
 512         if (tdc->desc)
 513                 tegra_adma_stop(tdc);
 514
 515         tegra_adma_request_free(tdc);
 516         vchan_get_all_descriptors(&tdc->vc, &head);
 517         spin_unlock_irqrestore(&tdc->vc.lock, flags);
 518         vchan_dma_desc_free_list(&tdc->vc, &head);
 519
 520         return 0;
 521 }
 522
 523 static enum dma_status tegra_adma_tx_status(struct dma_chan *dc,
 524                                             dma_cookie_t cookie,
 525                                             struct dma_tx_state *txstate)
 526 {
 527         struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
 528         struct tegra_adma_desc *desc;
 529         struct virt_dma_desc *vd;
 530         enum dma_status ret;
 531         unsigned long flags;
 532         unsigned int residual;
 533
 534         ret = dma_cookie_status(dc, cookie, txstate);
 535         if (ret == DMA_COMPLETE || !txstate)
 536                 return ret;
 537
 538         spin_lock_irqsave(&tdc->vc.lock, flags);
 539
 540         vd = vchan_find_desc(&tdc->vc, cookie);
 541         if (vd) {
 542                 desc = to_tegra_adma_desc(&vd->tx);
 543                 residual = desc->ch_regs.tc;
 544         } else if (tdc->desc && tdc->desc->vd.tx.cookie == cookie) {
 545                 residual = tegra_adma_get_residue(tdc);
 546         } else {
 547                 residual = 0;
 548         }
 549
 550         spin_unlock_irqrestore(&tdc->vc.lock, flags);
 551
 552         dma_set_residue(txstate, residual);
 553
 554         return ret;
 555 }
 556
 557 static unsigned int tegra210_adma_get_burst_config(unsigned int burst_size)
 558 {
 559         if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE)
 560                 burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE;
 561
 562         return fls(burst_size) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT;
 563 }
 564
 565 static unsigned int tegra186_adma_get_burst_config(unsigned int burst_size)
 566 {
 567         if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE)
 568                 burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE;
 569
 570         return (burst_size - 1) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT;
 571 }
 572
 573 static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc,
 574                                       struct tegra_adma_desc *desc,
 575                                       dma_addr_t buf_addr,
 576                                       enum dma_transfer_direction direction)
 577 {
 578         struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;
 579         const struct tegra_adma_chip_data *cdata = tdc->tdma->cdata;
 580         unsigned int burst_size, adma_dir;
 581
 582         if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS)
 583                 return -EINVAL;
 584
 585         switch (direction) {
 586         case DMA_MEM_TO_DEV:
 587                 adma_dir = ADMA_CH_CTRL_DIR_MEM2AHUB;
 588                 burst_size = tdc->sconfig.dst_maxburst;
 589                 ch_regs->config = ADMA_CH_CONFIG_SRC_BUF(desc->num_periods - 1);
 590                 ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index,
 591                                                       cdata->ch_req_mask,
 592                                                       cdata->ch_req_tx_shift);
 593                 ch_regs->src_addr = buf_addr;
 594                 break;
 595
 596         case DMA_DEV_TO_MEM:
 597                 adma_dir = ADMA_CH_CTRL_DIR_AHUB2MEM;
 598                 burst_size = tdc->sconfig.src_maxburst;
 599                 ch_regs->config = ADMA_CH_CONFIG_TRG_BUF(desc->num_periods - 1);
 600                 ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index,
 601                                                       cdata->ch_req_mask,
 602                                                       cdata->ch_req_rx_shift);
 603                 ch_regs->trg_addr = buf_addr;
 604                 break;
 605
 606         default:
 607                 dev_err(tdc2dev(tdc), "DMA direction is not supported\n");
 608                 return -EINVAL;
 609         }
 610
 611         ch_regs->ctrl |= ADMA_CH_CTRL_DIR(adma_dir) |
 612                          ADMA_CH_CTRL_MODE_CONTINUOUS |
 613                          ADMA_CH_CTRL_FLOWCTRL_EN;
 614         ch_regs->config |= cdata->adma_get_burst_config(burst_size);
 615         ch_regs->config |= ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1);
 616         ch_regs->fifo_ctrl = cdata->ch_fifo_ctrl;
 617         ch_regs->tc = desc->period_len & ADMA_CH_TC_COUNT_MASK;
 618
 619         return tegra_adma_request_alloc(tdc, direction);
 620 }
 621
 622 static struct dma_async_tx_descriptor *tegra_adma_prep_dma_cyclic(
 623         struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len,
 624         size_t period_len, enum dma_transfer_direction direction,
 625         unsigned long flags)
 626 {
 627         struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
 628         struct tegra_adma_desc *desc = NULL;
 629
 630         if (!buf_len || !period_len || period_len > ADMA_CH_TC_COUNT_MASK) {
 631                 dev_err(tdc2dev(tdc), "invalid buffer/period len\n");
 632                 return NULL;
 633         }
 634
 635         if (buf_len % period_len) {
 636                 dev_err(tdc2dev(tdc), "buf_len not a multiple of period_len\n");
 637                 return NULL;
 638         }
 639
 640         if (!IS_ALIGNED(buf_addr, 4)) {
 641                 dev_err(tdc2dev(tdc), "invalid buffer alignment\n");
 642                 return NULL;
 643         }
 644
 645         desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
 646         if (!desc)
 647                 return NULL;
 648
 649         desc->buf_len = buf_len;
 650         desc->period_len = period_len;
 651         desc->num_periods = buf_len / period_len;
 652
 653         if (tegra_adma_set_xfer_params(tdc, desc, buf_addr, direction)) {
 654                 kfree(desc);
 655                 return NULL;
 656         }
 657
 658         return vchan_tx_prep(&tdc->vc, &desc->vd, flags);
 659 }
 660
 661 static int tegra_adma_alloc_chan_resources(struct dma_chan *dc)
 662 {
 663         struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
 664         int ret;
 665
 666         ret = request_irq(tdc->irq, tegra_adma_isr, 0, dma_chan_name(dc), tdc);
 667         if (ret) {
 668                 dev_err(tdc2dev(tdc), "failed to get interrupt for %s\n",
 669                         dma_chan_name(dc));
 670                 return ret;
 671         }
 672
 673         ret = pm_runtime_get_sync(tdc2dev(tdc));
 674         if (ret < 0) {
 675                 free_irq(tdc->irq, tdc);
 676                 return ret;
 677         }
 678
 679         dma_cookie_init(&tdc->vc.chan);
 680
 681         return 0;
 682 }
 683
 684 static void tegra_adma_free_chan_resources(struct dma_chan *dc)
 685 {
 686         struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
 687
 688         tegra_adma_terminate_all(dc);
 689         vchan_free_chan_resources(&tdc->vc);
 690         tasklet_kill(&tdc->vc.task);
 691         free_irq(tdc->irq, tdc);
 692         pm_runtime_put(tdc2dev(tdc));
 693
 694         tdc->sreq_index = 0;
 695         tdc->sreq_dir = DMA_TRANS_NONE;
 696 }
 697
 698 static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec,
 699                                            struct of_dma *ofdma)
 700 {
 701         struct tegra_adma *tdma = ofdma->of_dma_data;
 702         struct tegra_adma_chan *tdc;
 703         struct dma_chan *chan;
 704         unsigned int sreq_index;
 705
 706         if (dma_spec->args_count != 1)
 707                 return NULL;
 708
 709         sreq_index = dma_spec->args[0];
 710
 711         if (sreq_index == 0) {
 712                 dev_err(tdma->dev, "DMA request must not be 0\n");
 713                 return NULL;
 714         }
 715
 716         chan = dma_get_any_slave_channel(&tdma->dma_dev);
 717         if (!chan)
 718                 return NULL;
 719
 720         tdc = to_tegra_adma_chan(chan);
 721         tdc->sreq_index = sreq_index;
 722
 723         return chan;
 724 }
 725
 726 static int tegra_adma_runtime_suspend(struct device *dev)
 727 {
 728         struct tegra_adma *tdma = dev_get_drvdata(dev);
 729         struct tegra_adma_chan_regs *ch_reg;
 730         struct tegra_adma_chan *tdc;
 731         int i;
 732
 733         tdma->global_cmd = tdma_read(tdma, ADMA_GLOBAL_CMD);
 734         if (!tdma->global_cmd)
 735                 goto clk_disable;
 736
 737         for (i = 0; i < tdma->nr_channels; i++) {
 738                 tdc = &tdma->channels[i];
 739                 ch_reg = &tdc->ch_regs;
 740                 ch_reg->cmd = tdma_ch_read(tdc, ADMA_CH_CMD);
 741                 /* skip if channel is not active */
 742                 if (!ch_reg->cmd)
 743                         continue;
 744                 ch_reg->tc = tdma_ch_read(tdc, ADMA_CH_TC);
 745                 ch_reg->src_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_SRC_ADDR);
 746                 ch_reg->trg_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_TRG_ADDR);
 747                 ch_reg->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);
 748                 ch_reg->fifo_ctrl = tdma_ch_read(tdc, ADMA_CH_FIFO_CTRL);
 749                 ch_reg->config = tdma_ch_read(tdc, ADMA_CH_CONFIG);
 750         }
 751
 752 clk_disable:
 753         clk_disable_unprepare(tdma->ahub_clk);
 754
 755         return 0;
 756 }
 757
 758 static int tegra_adma_runtime_resume(struct device *dev)
 759 {
 760         struct tegra_adma *tdma = dev_get_drvdata(dev);
 761         struct tegra_adma_chan_regs *ch_reg;
 762         struct tegra_adma_chan *tdc;
 763         int ret, i;
 764
 765         ret = clk_prepare_enable(tdma->ahub_clk);
 766         if (ret) {
 767                 dev_err(dev, "ahub clk_enable failed: %d\n", ret);
 768                 return ret;
 769         }
 770         tdma_write(tdma, ADMA_GLOBAL_CMD, tdma->global_cmd);
 771
 772         if (!tdma->global_cmd)
 773                 return 0;
 774
 775         for (i = 0; i < tdma->nr_channels; i++) {
 776                 tdc = &tdma->channels[i];
 777                 ch_reg = &tdc->ch_regs;
 778                 /* skip if channel was not active earlier */
 779                 if (!ch_reg->cmd)
 780                         continue;
 781                 tdma_ch_write(tdc, ADMA_CH_TC, ch_reg->tc);
 782                 tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, ch_reg->src_addr);
 783                 tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, ch_reg->trg_addr);
 784                 tdma_ch_write(tdc, ADMA_CH_CTRL, ch_reg->ctrl);
 785                 tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_reg->fifo_ctrl);
 786                 tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_reg->config);
 787                 tdma_ch_write(tdc, ADMA_CH_CMD, ch_reg->cmd);
 788         }
 789
 790         return 0;
 791 }
 792
 793 static const struct tegra_adma_chip_data tegra210_chip_data = {
 794         .adma_get_burst_config  = tegra210_adma_get_burst_config,
 795         .global_reg_offset      = 0xc00,
 796         .global_int_clear       = 0x20,
 797         .ch_req_tx_shift        = 28,
 798         .ch_req_rx_shift        = 24,
 799         .ch_base_offset         = 0,
 800         .ch_fifo_ctrl           = TEGRA210_FIFO_CTRL_DEFAULT,
 801         .ch_req_mask            = 0xf,
 802         .ch_req_max             = 10,
 803         .ch_reg_size            = 0x80,
 804         .nr_channels            = 22,
 805 };
 806
 807 static const struct tegra_adma_chip_data tegra186_chip_data = {
 808         .adma_get_burst_config  = tegra186_adma_get_burst_config,
 809         .global_reg_offset      = 0,
 810         .global_int_clear       = 0x402c,
 811         .ch_req_tx_shift        = 27,
 812         .ch_req_rx_shift        = 22,
 813         .ch_base_offset         = 0x10000,
 814         .ch_fifo_ctrl           = TEGRA186_FIFO_CTRL_DEFAULT,
 815         .ch_req_mask            = 0x1f,
 816         .ch_req_max             = 20,
 817         .ch_reg_size            = 0x100,
 818         .nr_channels            = 32,
 819 };
 820
 821 static const struct of_device_id tegra_adma_of_match[] = {
 822         { .compatible = "nvidia,tegra210-adma", .data = &tegra210_chip_data },
 823         { .compatible = "nvidia,tegra186-adma", .data = &tegra186_chip_data },
 824         { },
 825 };
 826 MODULE_DEVICE_TABLE(of, tegra_adma_of_match);
 827
 828 static int tegra_adma_probe(struct platform_device *pdev)
 829 {
 830         const struct tegra_adma_chip_data *cdata;
 831         struct tegra_adma *tdma;
 832         struct resource *res;
 833         int ret, i;
 834
 835         cdata = of_device_get_match_data(&pdev->dev);
 836         if (!cdata) {
 837                 dev_err(&pdev->dev, "device match data not found\n");
 838                 return -ENODEV;
 839         }
 840
 841         tdma = devm_kzalloc(&pdev->dev,
 842                             struct_size(tdma, channels, cdata->nr_channels),
 843                             GFP_KERNEL);
 844         if (!tdma)
 845                 return -ENOMEM;
 846
 847         tdma->dev = &pdev->dev;
 848         tdma->cdata = cdata;
 849         tdma->nr_channels = cdata->nr_channels;
 850         platform_set_drvdata(pdev, tdma);
 851
 852         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 853         tdma->base_addr = devm_ioremap_resource(&pdev->dev, res);
 854         if (IS_ERR(tdma->base_addr))
 855                 return PTR_ERR(tdma->base_addr);
 856
 857         tdma->ahub_clk = devm_clk_get(&pdev->dev, "d_audio");
 858         if (IS_ERR(tdma->ahub_clk)) {
 859                 dev_err(&pdev->dev, "Error: Missing ahub controller clock\n");
 860                 return PTR_ERR(tdma->ahub_clk);
 861         }
 862
 863         INIT_LIST_HEAD(&tdma->dma_dev.channels);
 864         for (i = 0; i < tdma->nr_channels; i++) {
 865                 struct tegra_adma_chan *tdc = &tdma->channels[i];
 866
 867                 tdc->chan_addr = tdma->base_addr + cdata->ch_base_offset
 868                                  + (cdata->ch_reg_size * i);
 869
 870                 tdc->irq = of_irq_get(pdev->dev.of_node, i);
 871                 if (tdc->irq <= 0) {
 872                         ret = tdc->irq ?: -ENXIO;
 873                         goto irq_dispose;
 874                 }
 875
 876                 vchan_init(&tdc->vc, &tdma->dma_dev);
 877                 tdc->vc.desc_free = tegra_adma_desc_free;
 878                 tdc->tdma = tdma;
 879         }
 880
 881         pm_runtime_enable(&pdev->dev);
 882
 883         ret = pm_runtime_get_sync(&pdev->dev);
 884         if (ret < 0)
 885                 goto rpm_disable;
 886
 887         ret = tegra_adma_init(tdma);
 888         if (ret)
 889                 goto rpm_put;
 890
 891         dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask);
 892         dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);
 893         dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask);
 894
 895         tdma->dma_dev.dev = &pdev->dev;
 896         tdma->dma_dev.device_alloc_chan_resources =
 897                                         tegra_adma_alloc_chan_resources;
 898         tdma->dma_dev.device_free_chan_resources =
 899                                         tegra_adma_free_chan_resources;
 900         tdma->dma_dev.device_issue_pending = tegra_adma_issue_pending;
 901         tdma->dma_dev.device_prep_dma_cyclic = tegra_adma_prep_dma_cyclic;
 902         tdma->dma_dev.device_config = tegra_adma_slave_config;
 903         tdma->dma_dev.device_tx_status = tegra_adma_tx_status;
 904         tdma->dma_dev.device_terminate_all = tegra_adma_terminate_all;
 905         tdma->dma_dev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
 906         tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
 907         tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 908         tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
 909         tdma->dma_dev.device_pause = tegra_adma_pause;
 910         tdma->dma_dev.device_resume = tegra_adma_resume;
 911
 912         ret = dma_async_device_register(&tdma->dma_dev);
 913         if (ret < 0) {
 914                 dev_err(&pdev->dev, "ADMA registration failed: %d\n", ret);
 915                 goto irq_dispose;
 916         }
 917
 918         ret = of_dma_controller_register(pdev->dev.of_node,
 919                                          tegra_dma_of_xlate, tdma);
 920         if (ret < 0) {
 921                 dev_err(&pdev->dev, "ADMA OF registration failed %d\n", ret);
 922                 goto dma_remove;
 923         }
 924
 925         pm_runtime_put(&pdev->dev);
 926
 927         dev_info(&pdev->dev, "Tegra210 ADMA driver registered %d channels\n",
 928                  tdma->nr_channels);
 929
 930         return 0;
 931
 932 dma_remove:
 933         dma_async_device_unregister(&tdma->dma_dev);
 934 rpm_put:
 935         pm_runtime_put_sync(&pdev->dev);
 936 rpm_disable:
 937         pm_runtime_disable(&pdev->dev);
 938 irq_dispose:
 939         while (--i >= 0)
 940                 irq_dispose_mapping(tdma->channels[i].irq);
 941
 942         return ret;
 943 }
 944
 945 static int tegra_adma_remove(struct platform_device *pdev)
 946 {
 947         struct tegra_adma *tdma = platform_get_drvdata(pdev);
 948         int i;
 949
 950         of_dma_controller_free(pdev->dev.of_node);
 951         dma_async_device_unregister(&tdma->dma_dev);
 952
 953         for (i = 0; i < tdma->nr_channels; ++i)
 954                 irq_dispose_mapping(tdma->channels[i].irq);
 955
 956         pm_runtime_put_sync(&pdev->dev);
 957         pm_runtime_disable(&pdev->dev);
 958
 959         return 0;
 960 }
 961
 962 static const struct dev_pm_ops tegra_adma_dev_pm_ops = {
 963         SET_RUNTIME_PM_OPS(tegra_adma_runtime_suspend,
 964                            tegra_adma_runtime_resume, NULL)
 965         SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
 966                                      pm_runtime_force_resume)
 967 };
 968
 969 static struct platform_driver tegra_admac_driver = {
 970         .driver = {
 971                 .name   = "tegra-adma",
 972                 .pm     = &tegra_adma_dev_pm_ops,
 973                 .of_match_table = tegra_adma_of_match,
 974         },
 975         .probe          = tegra_adma_probe,
 976         .remove         = tegra_adma_remove,
 977 };
 978
 979 module_platform_driver(tegra_admac_driver);
 980
 981 MODULE_ALIAS("platform:tegra210-adma");
 982 MODULE_DESCRIPTION("NVIDIA Tegra ADMA driver");
 983 MODULE_AUTHOR("Dara Ramesh <dramesh@nvidia.com>");
 984 MODULE_AUTHOR("Jon Hunter <jonathanh@nvidia.com>");
 985 MODULE_LICENSE("GPL v2");