drivers/gpu/drm/msm/msm_gpu.c

   1 /*
   2  * Copyright (C) 2013 Red Hat
   3  * Author: Rob Clark <robdclark@gmail.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 as published by
   7  * the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include "msm_gpu.h"
  19 #include "msm_gem.h"
  20 #include "msm_mmu.h"
  21 #include "msm_fence.h"
  22
  23
  24 /*
  25  * Power Management:
  26  */
  27
  28 #ifdef DOWNSTREAM_CONFIG_MSM_BUS_SCALING
  29 #include <mach/board.h>
  30 static void bs_init(struct msm_gpu *gpu)
  31 {
  32         if (gpu->bus_scale_table) {
  33                 gpu->bsc = msm_bus_scale_register_client(gpu->bus_scale_table);
  34                 DBG("bus scale client: %08x", gpu->bsc);
  35         }
  36 }
  37
  38 static void bs_fini(struct msm_gpu *gpu)
  39 {
  40         if (gpu->bsc) {
  41                 msm_bus_scale_unregister_client(gpu->bsc);
  42                 gpu->bsc = 0;
  43         }
  44 }
  45
  46 static void bs_set(struct msm_gpu *gpu, int idx)
  47 {
  48         if (gpu->bsc) {
  49                 DBG("set bus scaling: %d", idx);
  50                 msm_bus_scale_client_update_request(gpu->bsc, idx);
  51         }
  52 }
  53 #else
  54 static void bs_init(struct msm_gpu *gpu) {}
  55 static void bs_fini(struct msm_gpu *gpu) {}
  56 static void bs_set(struct msm_gpu *gpu, int idx) {}
  57 #endif
  58
  59 static int enable_pwrrail(struct msm_gpu *gpu)
  60 {
  61         struct drm_device *dev = gpu->dev;
  62         int ret = 0;
  63
  64         if (gpu->gpu_reg) {
  65                 ret = regulator_enable(gpu->gpu_reg);
  66                 if (ret) {
  67                         dev_err(dev->dev, "failed to enable 'gpu_reg': %d\n", ret);
  68                         return ret;
  69                 }
  70         }
  71
  72         if (gpu->gpu_cx) {
  73                 ret = regulator_enable(gpu->gpu_cx);
  74                 if (ret) {
  75                         dev_err(dev->dev, "failed to enable 'gpu_cx': %d\n", ret);
  76                         return ret;
  77                 }
  78         }
  79
  80         return 0;
  81 }
  82
  83 static int disable_pwrrail(struct msm_gpu *gpu)
  84 {
  85         if (gpu->gpu_cx)
  86                 regulator_disable(gpu->gpu_cx);
  87         if (gpu->gpu_reg)
  88                 regulator_disable(gpu->gpu_reg);
  89         return 0;
  90 }
  91
  92 static int enable_clk(struct msm_gpu *gpu)
  93 {
  94         int i;
  95
  96         if (gpu->core_clk && gpu->fast_rate)
  97                 clk_set_rate(gpu->core_clk, gpu->fast_rate);
  98
  99         /* Set the RBBM timer rate to 19.2Mhz */
 100         if (gpu->rbbmtimer_clk)
 101                 clk_set_rate(gpu->rbbmtimer_clk, 19200000);
 102
 103         for (i = gpu->nr_clocks - 1; i >= 0; i--)
 104                 if (gpu->grp_clks[i])
 105                         clk_prepare(gpu->grp_clks[i]);
 106
 107         for (i = gpu->nr_clocks - 1; i >= 0; i--)
 108                 if (gpu->grp_clks[i])
 109                         clk_enable(gpu->grp_clks[i]);
 110
 111         return 0;
 112 }
 113
 114 static int disable_clk(struct msm_gpu *gpu)
 115 {
 116         int i;
 117
 118         for (i = gpu->nr_clocks - 1; i >= 0; i--)
 119                 if (gpu->grp_clks[i])
 120                         clk_disable(gpu->grp_clks[i]);
 121
 122         for (i = gpu->nr_clocks - 1; i >= 0; i--)
 123                 if (gpu->grp_clks[i])
 124                         clk_unprepare(gpu->grp_clks[i]);
 125
 126         /*
 127          * Set the clock to a deliberately low rate. On older targets the clock
 128          * speed had to be non zero to avoid problems. On newer targets this
 129          * will be rounded down to zero anyway so it all works out.
 130          */
 131         if (gpu->core_clk)
 132                 clk_set_rate(gpu->core_clk, 27000000);
 133
 134         if (gpu->rbbmtimer_clk)
 135                 clk_set_rate(gpu->rbbmtimer_clk, 0);
 136
 137         return 0;
 138 }
 139
 140 static int enable_axi(struct msm_gpu *gpu)
 141 {
 142         if (gpu->ebi1_clk)
 143                 clk_prepare_enable(gpu->ebi1_clk);
 144         if (gpu->bus_freq)
 145                 bs_set(gpu, gpu->bus_freq);
 146         return 0;
 147 }
 148
 149 static int disable_axi(struct msm_gpu *gpu)
 150 {
 151         if (gpu->ebi1_clk)
 152                 clk_disable_unprepare(gpu->ebi1_clk);
 153         if (gpu->bus_freq)
 154                 bs_set(gpu, 0);
 155         return 0;
 156 }
 157
 158 int msm_gpu_pm_resume(struct msm_gpu *gpu)
 159 {
 160         int ret;
 161
 162         DBG("%s", gpu->name);
 163
 164         ret = enable_pwrrail(gpu);
 165         if (ret)
 166                 return ret;
 167
 168         ret = enable_clk(gpu);
 169         if (ret)
 170                 return ret;
 171
 172         ret = enable_axi(gpu);
 173         if (ret)
 174                 return ret;
 175
 176         gpu->needs_hw_init = true;
 177
 178         return 0;
 179 }
 180
 181 int msm_gpu_pm_suspend(struct msm_gpu *gpu)
 182 {
 183         int ret;
 184
 185         DBG("%s", gpu->name);
 186
 187         ret = disable_axi(gpu);
 188         if (ret)
 189                 return ret;
 190
 191         ret = disable_clk(gpu);
 192         if (ret)
 193                 return ret;
 194
 195         ret = disable_pwrrail(gpu);
 196         if (ret)
 197                 return ret;
 198
 199         return 0;
 200 }
 201
 202 int msm_gpu_hw_init(struct msm_gpu *gpu)
 203 {
 204         int ret;
 205
 206         WARN_ON(!mutex_is_locked(&gpu->dev->struct_mutex));
 207
 208         if (!gpu->needs_hw_init)
 209                 return 0;
 210
 211         disable_irq(gpu->irq);
 212         ret = gpu->funcs->hw_init(gpu);
 213         if (!ret)
 214                 gpu->needs_hw_init = false;
 215         enable_irq(gpu->irq);
 216
 217         return ret;
 218 }
 219
 220 /*
 221  * Hangcheck detection for locked gpu:
 222  */
 223
 224 static void update_fences(struct msm_gpu *gpu, struct msm_ringbuffer *ring,
 225                 uint32_t fence)
 226 {
 227         struct msm_gem_submit *submit;
 228
 229         list_for_each_entry(submit, &ring->submits, node) {
 230                 if (submit->seqno > fence)
 231                         break;
 232
 233                 msm_update_fence(submit->ring->fctx,
 234                         submit->fence->seqno);
 235         }
 236 }
 237
 238 static void retire_submits(struct msm_gpu *gpu);
 239
 240 static void recover_worker(struct work_struct *work)
 241 {
 242         struct msm_gpu *gpu = container_of(work, struct msm_gpu, recover_work);
 243         struct drm_device *dev = gpu->dev;
 244         struct msm_gem_submit *submit;
 245         struct msm_ringbuffer *cur_ring = gpu->funcs->active_ring(gpu);
 246         uint64_t fence;
 247         int i;
 248
 249         /* Update all the rings with the latest and greatest fence */
 250         for (i = 0; i < ARRAY_SIZE(gpu->rb); i++) {
 251                 struct msm_ringbuffer *ring = gpu->rb[i];
 252
 253                 fence = ring->memptrs->fence;
 254
 255                 /*
 256                  * For the current (faulting?) ring/submit advance the fence by
 257                  * one more to clear the faulting submit
 258                  */
 259                 if (ring == cur_ring)
 260                         fence = fence + 1;
 261
 262                 update_fences(gpu, ring, fence);
 263         }
 264
 265         mutex_lock(&dev->struct_mutex);
 266
 267
 268         dev_err(dev->dev, "%s: hangcheck recover!\n", gpu->name);
 269         fence = cur_ring->memptrs->fence + 1;
 270
 271         list_for_each_entry(submit, &cur_ring->submits, node) {
 272                 if (submit->seqno == fence) {
 273                         struct task_struct *task;
 274
 275                         rcu_read_lock();
 276                         task = pid_task(submit->pid, PIDTYPE_PID);
 277                         if (task) {
 278                                 dev_err(dev->dev, "%s: offending task: %s\n",
 279                                                 gpu->name, task->comm);
 280                         }
 281                         rcu_read_unlock();
 282                         break;
 283                 }
 284         }
 285
 286         if (msm_gpu_active(gpu)) {
 287                 /* retire completed submits, plus the one that hung: */
 288                 retire_submits(gpu);
 289
 290                 pm_runtime_get_sync(&gpu->pdev->dev);
 291                 gpu->funcs->recover(gpu);
 292                 pm_runtime_put_sync(&gpu->pdev->dev);
 293
 294                 /*
 295                  * Replay all remaining submits starting with highest priority
 296                  * ring
 297                  */
 298                 for (i = 0; i < gpu->nr_rings; i++) {
 299                         struct msm_ringbuffer *ring = gpu->rb[i];
 300
 301                         list_for_each_entry(submit, &ring->submits, node)
 302                                 gpu->funcs->submit(gpu, submit, NULL);
 303                 }
 304         }
 305
 306         mutex_unlock(&dev->struct_mutex);
 307
 308         msm_gpu_retire(gpu);
 309 }
 310
 311 static void hangcheck_timer_reset(struct msm_gpu *gpu)
 312 {
 313         DBG("%s", gpu->name);
 314         mod_timer(&gpu->hangcheck_timer,
 315                         round_jiffies_up(jiffies + DRM_MSM_HANGCHECK_JIFFIES));
 316 }
 317
 318 static void hangcheck_handler(unsigned long data)
 319 {
 320         struct msm_gpu *gpu = (struct msm_gpu *)data;
 321         struct drm_device *dev = gpu->dev;
 322         struct msm_drm_private *priv = dev->dev_private;
 323         struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
 324         uint32_t fence = ring->memptrs->fence;
 325
 326         if (fence != ring->hangcheck_fence) {
 327                 /* some progress has been made.. ya! */
 328                 ring->hangcheck_fence = fence;
 329         } else if (fence < ring->seqno) {
 330                 /* no progress and not done.. hung! */
 331                 ring->hangcheck_fence = fence;
 332                 dev_err(dev->dev, "%s: hangcheck detected gpu lockup rb %d!\n",
 333                                 gpu->name, ring->id);
 334                 dev_err(dev->dev, "%s:     completed fence: %u\n",
 335                                 gpu->name, fence);
 336                 dev_err(dev->dev, "%s:     submitted fence: %u\n",
 337                                 gpu->name, ring->seqno);
 338
 339                 queue_work(priv->wq, &gpu->recover_work);
 340         }
 341
 342         /* if still more pending work, reset the hangcheck timer: */
 343         if (ring->seqno > ring->hangcheck_fence)
 344                 hangcheck_timer_reset(gpu);
 345
 346         /* workaround for missing irq: */
 347         queue_work(priv->wq, &gpu->retire_work);
 348 }
 349
 350 /*
 351  * Performance Counters:
 352  */
 353
 354 /* called under perf_lock */
 355 static int update_hw_cntrs(struct msm_gpu *gpu, uint32_t ncntrs, uint32_t *cntrs)
 356 {
 357         uint32_t current_cntrs[ARRAY_SIZE(gpu->last_cntrs)];
 358         int i, n = min(ncntrs, gpu->num_perfcntrs);
 359
 360         /* read current values: */
 361         for (i = 0; i < gpu->num_perfcntrs; i++)
 362                 current_cntrs[i] = gpu_read(gpu, gpu->perfcntrs[i].sample_reg);
 363
 364         /* update cntrs: */
 365         for (i = 0; i < n; i++)
 366                 cntrs[i] = current_cntrs[i] - gpu->last_cntrs[i];
 367
 368         /* save current values: */
 369         for (i = 0; i < gpu->num_perfcntrs; i++)
 370                 gpu->last_cntrs[i] = current_cntrs[i];
 371
 372         return n;
 373 }
 374
 375 static void update_sw_cntrs(struct msm_gpu *gpu)
 376 {
 377         ktime_t time;
 378         uint32_t elapsed;
 379         unsigned long flags;
 380
 381         spin_lock_irqsave(&gpu->perf_lock, flags);
 382         if (!gpu->perfcntr_active)
 383                 goto out;
 384
 385         time = ktime_get();
 386         elapsed = ktime_to_us(ktime_sub(time, gpu->last_sample.time));
 387
 388         gpu->totaltime += elapsed;
 389         if (gpu->last_sample.active)
 390                 gpu->activetime += elapsed;
 391
 392         gpu->last_sample.active = msm_gpu_active(gpu);
 393         gpu->last_sample.time = time;
 394
 395 out:
 396         spin_unlock_irqrestore(&gpu->perf_lock, flags);
 397 }
 398
 399 void msm_gpu_perfcntr_start(struct msm_gpu *gpu)
 400 {
 401         unsigned long flags;
 402
 403         pm_runtime_get_sync(&gpu->pdev->dev);
 404
 405         spin_lock_irqsave(&gpu->perf_lock, flags);
 406         /* we could dynamically enable/disable perfcntr registers too.. */
 407         gpu->last_sample.active = msm_gpu_active(gpu);
 408         gpu->last_sample.time = ktime_get();
 409         gpu->activetime = gpu->totaltime = 0;
 410         gpu->perfcntr_active = true;
 411         update_hw_cntrs(gpu, 0, NULL);
 412         spin_unlock_irqrestore(&gpu->perf_lock, flags);
 413 }
 414
 415 void msm_gpu_perfcntr_stop(struct msm_gpu *gpu)
 416 {
 417         gpu->perfcntr_active = false;
 418         pm_runtime_put_sync(&gpu->pdev->dev);
 419 }
 420
 421 /* returns -errno or # of cntrs sampled */
 422 int msm_gpu_perfcntr_sample(struct msm_gpu *gpu, uint32_t *activetime,
 423                 uint32_t *totaltime, uint32_t ncntrs, uint32_t *cntrs)
 424 {
 425         unsigned long flags;
 426         int ret;
 427
 428         spin_lock_irqsave(&gpu->perf_lock, flags);
 429
 430         if (!gpu->perfcntr_active) {
 431                 ret = -EINVAL;
 432                 goto out;
 433         }
 434
 435         *activetime = gpu->activetime;
 436         *totaltime = gpu->totaltime;
 437
 438         gpu->activetime = gpu->totaltime = 0;
 439
 440         ret = update_hw_cntrs(gpu, ncntrs, cntrs);
 441
 442 out:
 443         spin_unlock_irqrestore(&gpu->perf_lock, flags);
 444
 445         return ret;
 446 }
 447
 448 /*
 449  * Cmdstream submission/retirement:
 450  */
 451
 452 static void retire_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
 453 {
 454         int i;
 455
 456         for (i = 0; i < submit->nr_bos; i++) {
 457                 struct msm_gem_object *msm_obj = submit->bos[i].obj;
 458                 /* move to inactive: */
 459                 msm_gem_move_to_inactive(&msm_obj->base);
 460                 msm_gem_put_iova(&msm_obj->base, gpu->aspace);
 461                 drm_gem_object_unreference(&msm_obj->base);
 462         }
 463
 464         pm_runtime_mark_last_busy(&gpu->pdev->dev);
 465         pm_runtime_put_autosuspend(&gpu->pdev->dev);
 466         msm_gem_submit_free(submit);
 467 }
 468
 469 static void retire_submits(struct msm_gpu *gpu)
 470 {
 471         struct drm_device *dev = gpu->dev;
 472         struct msm_gem_submit *submit, *tmp;
 473         int i;
 474
 475         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 476
 477         /* Retire the commits starting with highest priority */
 478         for (i = 0; i < gpu->nr_rings; i++) {
 479                 struct msm_ringbuffer *ring = gpu->rb[i];
 480
 481                 list_for_each_entry_safe(submit, tmp, &ring->submits, node) {
 482                         if (dma_fence_is_signaled(submit->fence))
 483                                 retire_submit(gpu, submit);
 484                 }
 485         }
 486 }
 487
 488 static void retire_worker(struct work_struct *work)
 489 {
 490         struct msm_gpu *gpu = container_of(work, struct msm_gpu, retire_work);
 491         struct drm_device *dev = gpu->dev;
 492         int i;
 493
 494         for (i = 0; i < gpu->nr_rings; i++)
 495                 update_fences(gpu, gpu->rb[i], gpu->rb[i]->memptrs->fence);
 496
 497         mutex_lock(&dev->struct_mutex);
 498         retire_submits(gpu);
 499         mutex_unlock(&dev->struct_mutex);
 500 }
 501
 502 /* call from irq handler to schedule work to retire bo's */
 503 void msm_gpu_retire(struct msm_gpu *gpu)
 504 {
 505         struct msm_drm_private *priv = gpu->dev->dev_private;
 506         queue_work(priv->wq, &gpu->retire_work);
 507         update_sw_cntrs(gpu);
 508 }
 509
 510 /* add bo's to gpu's ring, and kick gpu: */
 511 void msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 512                 struct msm_file_private *ctx)
 513 {
 514         struct drm_device *dev = gpu->dev;
 515         struct msm_drm_private *priv = dev->dev_private;
 516         struct msm_ringbuffer *ring = submit->ring;
 517         int i;
 518
 519         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 520
 521         pm_runtime_get_sync(&gpu->pdev->dev);
 522
 523         msm_gpu_hw_init(gpu);
 524
 525         submit->seqno = ++ring->seqno;
 526
 527         list_add_tail(&submit->node, &ring->submits);
 528
 529         msm_rd_dump_submit(submit);
 530
 531         update_sw_cntrs(gpu);
 532
 533         for (i = 0; i < submit->nr_bos; i++) {
 534                 struct msm_gem_object *msm_obj = submit->bos[i].obj;
 535                 uint64_t iova;
 536
 537                 /* can't happen yet.. but when we add 2d support we'll have
 538                  * to deal w/ cross-ring synchronization:
 539                  */
 540                 WARN_ON(is_active(msm_obj) && (msm_obj->gpu != gpu));
 541
 542                 /* submit takes a reference to the bo and iova until retired: */
 543                 drm_gem_object_reference(&msm_obj->base);
 544                 msm_gem_get_iova(&msm_obj->base,
 545                                 submit->gpu->aspace, &iova);
 546
 547                 if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE)
 548                         msm_gem_move_to_active(&msm_obj->base, gpu, true, submit->fence);
 549                 else if (submit->bos[i].flags & MSM_SUBMIT_BO_READ)
 550                         msm_gem_move_to_active(&msm_obj->base, gpu, false, submit->fence);
 551         }
 552
 553         gpu->funcs->submit(gpu, submit, ctx);
 554         priv->lastctx = ctx;
 555
 556         hangcheck_timer_reset(gpu);
 557 }
 558
 559 /*
 560  * Init/Cleanup:
 561  */
 562
 563 static irqreturn_t irq_handler(int irq, void *data)
 564 {
 565         struct msm_gpu *gpu = data;
 566         return gpu->funcs->irq(gpu);
 567 }
 568
 569 static struct clk *get_clock(struct device *dev, const char *name)
 570 {
 571         struct clk *clk = devm_clk_get(dev, name);
 572
 573         return IS_ERR(clk) ? NULL : clk;
 574 }
 575
 576 static int get_clocks(struct platform_device *pdev, struct msm_gpu *gpu)
 577 {
 578         struct device *dev = &pdev->dev;
 579         struct property *prop;
 580         const char *name;
 581         int i = 0;
 582
 583         gpu->nr_clocks = of_property_count_strings(dev->of_node, "clock-names");
 584         if (gpu->nr_clocks < 1) {
 585                 gpu->nr_clocks = 0;
 586                 return 0;
 587         }
 588
 589         gpu->grp_clks = devm_kcalloc(dev, sizeof(struct clk *), gpu->nr_clocks,
 590                 GFP_KERNEL);
 591         if (!gpu->grp_clks)
 592                 return -ENOMEM;
 593
 594         of_property_for_each_string(dev->of_node, "clock-names", prop, name) {
 595                 gpu->grp_clks[i] = get_clock(dev, name);
 596
 597                 /* Remember the key clocks that we need to control later */
 598                 if (!strcmp(name, "core") || !strcmp(name, "core_clk"))
 599                         gpu->core_clk = gpu->grp_clks[i];
 600                 else if (!strcmp(name, "rbbmtimer") || !strcmp(name, "rbbmtimer_clk"))
 601                         gpu->rbbmtimer_clk = gpu->grp_clks[i];
 602
 603                 ++i;
 604         }
 605
 606         return 0;
 607 }
 608
 609 static struct msm_gem_address_space *
 610 msm_gpu_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev,
 611                 uint64_t va_start, uint64_t va_end)
 612 {
 613         struct iommu_domain *iommu;
 614         struct msm_gem_address_space *aspace;
 615         int ret;
 616
 617         /*
 618          * Setup IOMMU.. eventually we will (I think) do this once per context
 619          * and have separate page tables per context.  For now, to keep things
 620          * simple and to get something working, just use a single address space:
 621          */
 622         iommu = iommu_domain_alloc(&platform_bus_type);
 623         if (!iommu)
 624                 return NULL;
 625
 626         iommu->geometry.aperture_start = va_start;
 627         iommu->geometry.aperture_end = va_end;
 628
 629         dev_info(gpu->dev->dev, "%s: using IOMMU\n", gpu->name);
 630
 631         aspace = msm_gem_address_space_create(&pdev->dev, iommu, "gpu");
 632         if (IS_ERR(aspace)) {
 633                 dev_err(gpu->dev->dev, "failed to init iommu: %ld\n",
 634                         PTR_ERR(aspace));
 635                 iommu_domain_free(iommu);
 636                 return ERR_CAST(aspace);
 637         }
 638
 639         ret = aspace->mmu->funcs->attach(aspace->mmu, NULL, 0);
 640         if (ret) {
 641                 msm_gem_address_space_put(aspace);
 642                 return ERR_PTR(ret);
 643         }
 644
 645         return aspace;
 646 }
 647
 648 int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 649                 struct msm_gpu *gpu, const struct msm_gpu_funcs *funcs,
 650                 const char *name, struct msm_gpu_config *config)
 651 {
 652         int i, ret, nr_rings = config->nr_rings;
 653         void *memptrs;
 654         uint64_t memptrs_iova;
 655
 656         if (WARN_ON(gpu->num_perfcntrs > ARRAY_SIZE(gpu->last_cntrs)))
 657                 gpu->num_perfcntrs = ARRAY_SIZE(gpu->last_cntrs);
 658
 659         gpu->dev = drm;
 660         gpu->funcs = funcs;
 661         gpu->name = name;
 662
 663         INIT_LIST_HEAD(&gpu->active_list);
 664         INIT_WORK(&gpu->retire_work, retire_worker);
 665         INIT_WORK(&gpu->recover_work, recover_worker);
 666
 667
 668         setup_timer(&gpu->hangcheck_timer, hangcheck_handler,
 669                         (unsigned long)gpu);
 670
 671         spin_lock_init(&gpu->perf_lock);
 672
 673
 674         /* Map registers: */
 675         gpu->mmio = msm_ioremap(pdev, config->ioname, name);
 676         if (IS_ERR(gpu->mmio)) {
 677                 ret = PTR_ERR(gpu->mmio);
 678                 goto fail;
 679         }
 680
 681         /* Get Interrupt: */
 682         gpu->irq = platform_get_irq_byname(pdev, config->irqname);
 683         if (gpu->irq < 0) {
 684                 ret = gpu->irq;
 685                 dev_err(drm->dev, "failed to get irq: %d\n", ret);
 686                 goto fail;
 687         }
 688
 689         ret = devm_request_irq(&pdev->dev, gpu->irq, irq_handler,
 690                         IRQF_TRIGGER_HIGH, gpu->name, gpu);
 691         if (ret) {
 692                 dev_err(drm->dev, "failed to request IRQ%u: %d\n", gpu->irq, ret);
 693                 goto fail;
 694         }
 695
 696         ret = get_clocks(pdev, gpu);
 697         if (ret)
 698                 goto fail;
 699
 700         gpu->ebi1_clk = msm_clk_get(pdev, "bus");
 701         DBG("ebi1_clk: %p", gpu->ebi1_clk);
 702         if (IS_ERR(gpu->ebi1_clk))
 703                 gpu->ebi1_clk = NULL;
 704
 705         /* Acquire regulators: */
 706         gpu->gpu_reg = devm_regulator_get(&pdev->dev, "vdd");
 707         DBG("gpu_reg: %p", gpu->gpu_reg);
 708         if (IS_ERR(gpu->gpu_reg))
 709                 gpu->gpu_reg = NULL;
 710
 711         gpu->gpu_cx = devm_regulator_get(&pdev->dev, "vddcx");
 712         DBG("gpu_cx: %p", gpu->gpu_cx);
 713         if (IS_ERR(gpu->gpu_cx))
 714                 gpu->gpu_cx = NULL;
 715
 716         gpu->pdev = pdev;
 717         platform_set_drvdata(pdev, gpu);
 718
 719         bs_init(gpu);
 720
 721         gpu->aspace = msm_gpu_create_address_space(gpu, pdev,
 722                 config->va_start, config->va_end);
 723
 724         if (gpu->aspace == NULL)
 725                 dev_info(drm->dev, "%s: no IOMMU, fallback to VRAM carveout!\n", name);
 726         else if (IS_ERR(gpu->aspace)) {
 727                 ret = PTR_ERR(gpu->aspace);
 728                 goto fail;
 729         }
 730
 731         memptrs = msm_gem_kernel_new(drm, sizeof(*gpu->memptrs_bo),
 732                 MSM_BO_UNCACHED, gpu->aspace, &gpu->memptrs_bo,
 733                 &memptrs_iova);
 734
 735         if (IS_ERR(memptrs)) {
 736                 ret = PTR_ERR(memptrs);
 737                 dev_err(drm->dev, "could not allocate memptrs: %d\n", ret);
 738                 goto fail;
 739         }
 740
 741         if (nr_rings > ARRAY_SIZE(gpu->rb)) {
 742                 DRM_DEV_INFO_ONCE(drm->dev, "Only creating %lu ringbuffers\n",
 743                         ARRAY_SIZE(gpu->rb));
 744                 nr_rings = ARRAY_SIZE(gpu->rb);
 745         }
 746
 747         /* Create ringbuffer(s): */
 748         for (i = 0; i < nr_rings; i++) {
 749                 gpu->rb[i] = msm_ringbuffer_new(gpu, i, memptrs, memptrs_iova);
 750
 751                 if (IS_ERR(gpu->rb[i])) {
 752                         ret = PTR_ERR(gpu->rb[i]);
 753                         dev_err(drm->dev,
 754                                 "could not create ringbuffer %d: %d\n", i, ret);
 755                         goto fail;
 756                 }
 757
 758                 memptrs += sizeof(struct msm_rbmemptrs);
 759                 memptrs_iova += sizeof(struct msm_rbmemptrs);
 760         }
 761
 762         gpu->nr_rings = nr_rings;
 763
 764         return 0;
 765
 766 fail:
 767         for (i = 0; i < ARRAY_SIZE(gpu->rb); i++)  {
 768                 msm_ringbuffer_destroy(gpu->rb[i]);
 769                 gpu->rb[i] = NULL;
 770         }
 771
 772         if (gpu->memptrs_bo) {
 773                 msm_gem_put_vaddr(gpu->memptrs_bo);
 774                 msm_gem_put_iova(gpu->memptrs_bo, gpu->aspace);
 775                 drm_gem_object_unreference_unlocked(gpu->memptrs_bo);
 776         }
 777
 778         platform_set_drvdata(pdev, NULL);
 779         return ret;
 780 }
 781
 782 void msm_gpu_cleanup(struct msm_gpu *gpu)
 783 {
 784         int i;
 785
 786         DBG("%s", gpu->name);
 787
 788         WARN_ON(!list_empty(&gpu->active_list));
 789
 790         bs_fini(gpu);
 791
 792         for (i = 0; i < ARRAY_SIZE(gpu->rb); i++) {
 793                 msm_ringbuffer_destroy(gpu->rb[i]);
 794                 gpu->rb[i] = NULL;
 795         }
 796
 797         if (gpu->memptrs_bo) {
 798                 msm_gem_put_vaddr(gpu->memptrs_bo);
 799                 msm_gem_put_iova(gpu->memptrs_bo, gpu->aspace);
 800                 drm_gem_object_unreference_unlocked(gpu->memptrs_bo);
 801         }
 802
 803         if (!IS_ERR_OR_NULL(gpu->aspace)) {
 804                 gpu->aspace->mmu->funcs->detach(gpu->aspace->mmu,
 805                         NULL, 0);
 806                 msm_gem_address_space_put(gpu->aspace);
 807         }
 808 }