drivers/gpu/host1x/hw/cdma_hw.c

   1 /*
   2  * Tegra host1x Command DMA
   3  *
   4  * Copyright (c) 2010-2013, NVIDIA Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/slab.h>
  20 #include <linux/scatterlist.h>
  21 #include <linux/dma-mapping.h>
  22
  23 #include "../cdma.h"
  24 #include "../channel.h"
  25 #include "../dev.h"
  26 #include "../debug.h"
  27
  28 /*
  29  * Put the restart at the end of pushbuffer memory
  30  */
  31 static void push_buffer_init(struct push_buffer *pb)
  32 {
  33         *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0);
  34 }
  35
  36 /*
  37  * Increment timedout buffer's syncpt via CPU.
  38  */
  39 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
  40                                 u32 syncpt_incrs, u32 syncval, u32 nr_slots)
  41 {
  42         struct host1x *host1x = cdma_to_host1x(cdma);
  43         struct push_buffer *pb = &cdma->push_buffer;
  44         unsigned int i;
  45
  46         for (i = 0; i < syncpt_incrs; i++)
  47                 host1x_syncpt_incr(cdma->timeout.syncpt);
  48
  49         /* after CPU incr, ensure shadow is up to date */
  50         host1x_syncpt_load(cdma->timeout.syncpt);
  51
  52         /* NOP all the PB slots */
  53         while (nr_slots--) {
  54                 u32 *p = (u32 *)(pb->mapped + getptr);
  55                 *(p++) = HOST1X_OPCODE_NOP;
  56                 *(p++) = HOST1X_OPCODE_NOP;
  57                 dev_dbg(host1x->dev, "%s: NOP at %pad+%#x\n", __func__,
  58                         &pb->dma, getptr);
  59                 getptr = (getptr + 8) & (pb->size - 1);
  60         }
  61
  62         wmb();
  63 }
  64
  65 /*
  66  * Start channel DMA
  67  */
  68 static void cdma_start(struct host1x_cdma *cdma)
  69 {
  70         struct host1x_channel *ch = cdma_to_channel(cdma);
  71         u64 start, end;
  72
  73         if (cdma->running)
  74                 return;
  75
  76         cdma->last_pos = cdma->push_buffer.pos;
  77         start = cdma->push_buffer.dma;
  78         end = cdma->push_buffer.size + 4;
  79
  80         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
  81                          HOST1X_CHANNEL_DMACTRL);
  82
  83         /* set base, put and end pointer */
  84         host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);
  85 #if HOST1X_HW >= 6
  86         host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);
  87 #endif
  88         host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
  89 #if HOST1X_HW >= 6
  90         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMAPUT_HI);
  91 #endif
  92         host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);
  93 #if HOST1X_HW >= 6
  94         host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);
  95 #endif
  96
  97         /* reset GET */
  98         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
  99                          HOST1X_CHANNEL_DMACTRL_DMAGETRST |
 100                          HOST1X_CHANNEL_DMACTRL_DMAINITGET,
 101                          HOST1X_CHANNEL_DMACTRL);
 102
 103         /* start the command DMA */
 104         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
 105
 106         cdma->running = true;
 107 }
 108
 109 /*
 110  * Similar to cdma_start(), but rather than starting from an idle
 111  * state (where DMA GET is set to DMA PUT), on a timeout we restore
 112  * DMA GET from an explicit value (so DMA may again be pending).
 113  */
 114 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr)
 115 {
 116         struct host1x *host1x = cdma_to_host1x(cdma);
 117         struct host1x_channel *ch = cdma_to_channel(cdma);
 118         u64 start, end;
 119
 120         if (cdma->running)
 121                 return;
 122
 123         cdma->last_pos = cdma->push_buffer.pos;
 124
 125         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 126                          HOST1X_CHANNEL_DMACTRL);
 127
 128         start = cdma->push_buffer.dma;
 129         end = cdma->push_buffer.size + 4;
 130
 131         /* set base, end pointer (all of memory) */
 132         host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);
 133 #if HOST1X_HW >= 6
 134         host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);
 135 #endif
 136         host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);
 137 #if HOST1X_HW >= 6
 138         host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);
 139 #endif
 140
 141         /* set GET, by loading the value in PUT (then reset GET) */
 142         host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT);
 143         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
 144                          HOST1X_CHANNEL_DMACTRL_DMAGETRST |
 145                          HOST1X_CHANNEL_DMACTRL_DMAINITGET,
 146                          HOST1X_CHANNEL_DMACTRL);
 147
 148         dev_dbg(host1x->dev,
 149                 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__,
 150                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 151                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 152                 cdma->last_pos);
 153
 154         /* deassert GET reset and set PUT */
 155         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 156                          HOST1X_CHANNEL_DMACTRL);
 157         host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
 158
 159         /* start the command DMA */
 160         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
 161
 162         cdma->running = true;
 163 }
 164
 165 /*
 166  * Kick channel DMA into action by writing its PUT offset (if it has changed)
 167  */
 168 static void cdma_flush(struct host1x_cdma *cdma)
 169 {
 170         struct host1x_channel *ch = cdma_to_channel(cdma);
 171
 172         if (cdma->push_buffer.pos != cdma->last_pos) {
 173                 host1x_ch_writel(ch, cdma->push_buffer.pos,
 174                                  HOST1X_CHANNEL_DMAPUT);
 175                 cdma->last_pos = cdma->push_buffer.pos;
 176         }
 177 }
 178
 179 static void cdma_stop(struct host1x_cdma *cdma)
 180 {
 181         struct host1x_channel *ch = cdma_to_channel(cdma);
 182
 183         mutex_lock(&cdma->lock);
 184
 185         if (cdma->running) {
 186                 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
 187                 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 188                                  HOST1X_CHANNEL_DMACTRL);
 189                 cdma->running = false;
 190         }
 191
 192         mutex_unlock(&cdma->lock);
 193 }
 194
 195 static void cdma_hw_cmdproc_stop(struct host1x *host, struct host1x_channel *ch,
 196                                  bool stop)
 197 {
 198 #if HOST1X_HW >= 6
 199         host1x_ch_writel(ch, stop ? 0x1 : 0x0, HOST1X_CHANNEL_CMDPROC_STOP);
 200 #else
 201         u32 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP);
 202         if (stop)
 203                 cmdproc_stop |= BIT(ch->id);
 204         else
 205                 cmdproc_stop &= ~BIT(ch->id);
 206         host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 207 #endif
 208 }
 209
 210 static void cdma_hw_teardown(struct host1x *host, struct host1x_channel *ch)
 211 {
 212 #if HOST1X_HW >= 6
 213         host1x_ch_writel(ch, 0x1, HOST1X_CHANNEL_TEARDOWN);
 214 #else
 215         host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN);
 216 #endif
 217 }
 218
 219 /*
 220  * Stops both channel's command processor and CDMA immediately.
 221  * Also, tears down the channel and resets corresponding module.
 222  */
 223 static void cdma_freeze(struct host1x_cdma *cdma)
 224 {
 225         struct host1x *host = cdma_to_host1x(cdma);
 226         struct host1x_channel *ch = cdma_to_channel(cdma);
 227
 228         if (cdma->torndown && !cdma->running) {
 229                 dev_warn(host->dev, "Already torn down\n");
 230                 return;
 231         }
 232
 233         dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id);
 234
 235         cdma_hw_cmdproc_stop(host, ch, true);
 236
 237         dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
 238                 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 239                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 240                 cdma->last_pos);
 241
 242         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 243                          HOST1X_CHANNEL_DMACTRL);
 244
 245         cdma_hw_teardown(host, ch);
 246
 247         cdma->running = false;
 248         cdma->torndown = true;
 249 }
 250
 251 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
 252 {
 253         struct host1x *host1x = cdma_to_host1x(cdma);
 254         struct host1x_channel *ch = cdma_to_channel(cdma);
 255
 256         dev_dbg(host1x->dev,
 257                 "resuming channel (id %u, DMAGET restart = 0x%x)\n",
 258                 ch->id, getptr);
 259
 260         cdma_hw_cmdproc_stop(host1x, ch, false);
 261
 262         cdma->torndown = false;
 263         cdma_timeout_restart(cdma, getptr);
 264 }
 265
 266 /*
 267  * If this timeout fires, it indicates the current sync_queue entry has
 268  * exceeded its TTL and the userctx should be timed out and remaining
 269  * submits already issued cleaned up (future submits return an error).
 270  */
 271 static void cdma_timeout_handler(struct work_struct *work)
 272 {
 273         u32 syncpt_val;
 274         struct host1x_cdma *cdma;
 275         struct host1x *host1x;
 276         struct host1x_channel *ch;
 277
 278         cdma = container_of(to_delayed_work(work), struct host1x_cdma,
 279                             timeout.wq);
 280         host1x = cdma_to_host1x(cdma);
 281         ch = cdma_to_channel(cdma);
 282
 283         host1x_debug_dump(cdma_to_host1x(cdma));
 284
 285         mutex_lock(&cdma->lock);
 286
 287         if (!cdma->timeout.client) {
 288                 dev_dbg(host1x->dev,
 289                         "cdma_timeout: expired, but has no clientid\n");
 290                 mutex_unlock(&cdma->lock);
 291                 return;
 292         }
 293
 294         /* stop processing to get a clean snapshot */
 295         cdma_hw_cmdproc_stop(host1x, ch, true);
 296
 297         syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
 298
 299         /* has buffer actually completed? */
 300         if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {
 301                 dev_dbg(host1x->dev,
 302                         "cdma_timeout: expired, but buffer had completed\n");
 303                 /* restore */
 304                 cdma_hw_cmdproc_stop(host1x, ch, false);
 305                 mutex_unlock(&cdma->lock);
 306                 return;
 307         }
 308
 309         dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n",
 310                  __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
 311                  syncpt_val, cdma->timeout.syncpt_val);
 312
 313         /* stop HW, resetting channel/module */
 314         host1x_hw_cdma_freeze(host1x, cdma);
 315
 316         host1x_cdma_update_sync_queue(cdma, ch->dev);
 317         mutex_unlock(&cdma->lock);
 318 }
 319
 320 /*
 321  * Init timeout resources
 322  */
 323 static int cdma_timeout_init(struct host1x_cdma *cdma, unsigned int syncpt)
 324 {
 325         INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
 326         cdma->timeout.initialized = true;
 327
 328         return 0;
 329 }
 330
 331 /*
 332  * Clean up timeout resources
 333  */
 334 static void cdma_timeout_destroy(struct host1x_cdma *cdma)
 335 {
 336         if (cdma->timeout.initialized)
 337                 cancel_delayed_work(&cdma->timeout.wq);
 338
 339         cdma->timeout.initialized = false;
 340 }
 341
 342 static const struct host1x_cdma_ops host1x_cdma_ops = {
 343         .start = cdma_start,
 344         .stop = cdma_stop,
 345         .flush = cdma_flush,
 346
 347         .timeout_init = cdma_timeout_init,
 348         .timeout_destroy = cdma_timeout_destroy,
 349         .freeze = cdma_freeze,
 350         .resume = cdma_resume,
 351         .timeout_cpu_incr = cdma_timeout_cpu_incr,
 352 };
 353
 354 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = {
 355         .init = push_buffer_init,
 356 };