drivers/gpu/host1x/job.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Tegra host1x Job
   4  *
   5  * Copyright (c) 2010-2015, NVIDIA Corporation.
   6  */
   7
   8 #include <linux/dma-mapping.h>
   9 #include <linux/err.h>
  10 #include <linux/host1x.h>
  11 #include <linux/kref.h>
  12 #include <linux/module.h>
  13 #include <linux/scatterlist.h>
  14 #include <linux/slab.h>
  15 #include <linux/vmalloc.h>
  16 #include <trace/events/host1x.h>
  17
  18 #include "channel.h"
  19 #include "dev.h"
  20 #include "job.h"
  21 #include "syncpt.h"
  22
  23 #define HOST1X_WAIT_SYNCPT_OFFSET 0x8
  24
  25 struct host1x_job *host1x_job_alloc(struct host1x_channel *ch,
  26                                     u32 num_cmdbufs, u32 num_relocs)
  27 {
  28         struct host1x_job *job = NULL;
  29         unsigned int num_unpins = num_cmdbufs + num_relocs;
  30         u64 total;
  31         void *mem;
  32
  33         /* Check that we're not going to overflow */
  34         total = sizeof(struct host1x_job) +
  35                 (u64)num_relocs * sizeof(struct host1x_reloc) +
  36                 (u64)num_unpins * sizeof(struct host1x_job_unpin_data) +
  37                 (u64)num_cmdbufs * sizeof(struct host1x_job_gather) +
  38                 (u64)num_unpins * sizeof(dma_addr_t) +
  39                 (u64)num_unpins * sizeof(u32 *);
  40         if (total > ULONG_MAX)
  41                 return NULL;
  42
  43         mem = job = kzalloc(total, GFP_KERNEL);
  44         if (!job)
  45                 return NULL;
  46
  47         kref_init(&job->ref);
  48         job->channel = ch;
  49
  50         /* Redistribute memory to the structs  */
  51         mem += sizeof(struct host1x_job);
  52         job->relocs = num_relocs ? mem : NULL;
  53         mem += num_relocs * sizeof(struct host1x_reloc);
  54         job->unpins = num_unpins ? mem : NULL;
  55         mem += num_unpins * sizeof(struct host1x_job_unpin_data);
  56         job->gathers = num_cmdbufs ? mem : NULL;
  57         mem += num_cmdbufs * sizeof(struct host1x_job_gather);
  58         job->addr_phys = num_unpins ? mem : NULL;
  59
  60         job->reloc_addr_phys = job->addr_phys;
  61         job->gather_addr_phys = &job->addr_phys[num_relocs];
  62
  63         return job;
  64 }
  65 EXPORT_SYMBOL(host1x_job_alloc);
  66
  67 struct host1x_job *host1x_job_get(struct host1x_job *job)
  68 {
  69         kref_get(&job->ref);
  70         return job;
  71 }
  72 EXPORT_SYMBOL(host1x_job_get);
  73
  74 static void job_free(struct kref *ref)
  75 {
  76         struct host1x_job *job = container_of(ref, struct host1x_job, ref);
  77
  78         kfree(job);
  79 }
  80
  81 void host1x_job_put(struct host1x_job *job)
  82 {
  83         kref_put(&job->ref, job_free);
  84 }
  85 EXPORT_SYMBOL(host1x_job_put);
  86
  87 void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo,
  88                            unsigned int words, unsigned int offset)
  89 {
  90         struct host1x_job_gather *gather = &job->gathers[job->num_gathers];
  91
  92         gather->words = words;
  93         gather->bo = bo;
  94         gather->offset = offset;
  95
  96         job->num_gathers++;
  97 }
  98 EXPORT_SYMBOL(host1x_job_add_gather);
  99
 100 static unsigned int pin_job(struct host1x *host, struct host1x_job *job)
 101 {
 102         struct device *dev = job->client->dev;
 103         unsigned int i;
 104         int err;
 105
 106         job->num_unpins = 0;
 107
 108         for (i = 0; i < job->num_relocs; i++) {
 109                 struct host1x_reloc *reloc = &job->relocs[i];
 110                 struct sg_table *sgt;
 111                 dma_addr_t phys_addr;
 112
 113                 reloc->target.bo = host1x_bo_get(reloc->target.bo);
 114                 if (!reloc->target.bo) {
 115                         err = -EINVAL;
 116                         goto unpin;
 117                 }
 118
 119                 sgt = host1x_bo_pin(dev, reloc->target.bo, &phys_addr);
 120                 if (IS_ERR(sgt)) {
 121                         err = PTR_ERR(sgt);
 122                         goto unpin;
 123                 }
 124
 125                 job->addr_phys[job->num_unpins] = phys_addr;
 126                 job->unpins[job->num_unpins].bo = reloc->target.bo;
 127                 job->unpins[job->num_unpins].sgt = sgt;
 128                 job->num_unpins++;
 129         }
 130
 131         for (i = 0; i < job->num_gathers; i++) {
 132                 struct host1x_job_gather *g = &job->gathers[i];
 133                 size_t gather_size = 0;
 134                 struct scatterlist *sg;
 135                 struct sg_table *sgt;
 136                 dma_addr_t phys_addr;
 137                 unsigned long shift;
 138                 struct iova *alloc;
 139                 unsigned int j;
 140
 141                 g->bo = host1x_bo_get(g->bo);
 142                 if (!g->bo) {
 143                         err = -EINVAL;
 144                         goto unpin;
 145                 }
 146
 147                 sgt = host1x_bo_pin(host->dev, g->bo, &phys_addr);
 148                 if (IS_ERR(sgt)) {
 149                         err = PTR_ERR(sgt);
 150                         goto unpin;
 151                 }
 152
 153                 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && host->domain) {
 154                         for_each_sg(sgt->sgl, sg, sgt->nents, j)
 155                                 gather_size += sg->length;
 156                         gather_size = iova_align(&host->iova, gather_size);
 157
 158                         shift = iova_shift(&host->iova);
 159                         alloc = alloc_iova(&host->iova, gather_size >> shift,
 160                                            host->iova_end >> shift, true);
 161                         if (!alloc) {
 162                                 err = -ENOMEM;
 163                                 goto unpin;
 164                         }
 165
 166                         err = iommu_map_sg(host->domain,
 167                                         iova_dma_addr(&host->iova, alloc),
 168                                         sgt->sgl, sgt->nents, IOMMU_READ);
 169                         if (err == 0) {
 170                                 __free_iova(&host->iova, alloc);
 171                                 err = -EINVAL;
 172                                 goto unpin;
 173                         }
 174
 175                         job->addr_phys[job->num_unpins] =
 176                                 iova_dma_addr(&host->iova, alloc);
 177                         job->unpins[job->num_unpins].size = gather_size;
 178                 } else {
 179                         job->addr_phys[job->num_unpins] = phys_addr;
 180                 }
 181
 182                 job->gather_addr_phys[i] = job->addr_phys[job->num_unpins];
 183
 184                 job->unpins[job->num_unpins].bo = g->bo;
 185                 job->unpins[job->num_unpins].sgt = sgt;
 186                 job->num_unpins++;
 187         }
 188
 189         return 0;
 190
 191 unpin:
 192         host1x_job_unpin(job);
 193         return err;
 194 }
 195
 196 static int do_relocs(struct host1x_job *job, struct host1x_job_gather *g)
 197 {
 198         u32 last_page = ~0;
 199         void *cmdbuf_page_addr = NULL;
 200         struct host1x_bo *cmdbuf = g->bo;
 201         unsigned int i;
 202
 203         /* pin & patch the relocs for one gather */
 204         for (i = 0; i < job->num_relocs; i++) {
 205                 struct host1x_reloc *reloc = &job->relocs[i];
 206                 u32 reloc_addr = (job->reloc_addr_phys[i] +
 207                                   reloc->target.offset) >> reloc->shift;
 208                 u32 *target;
 209
 210                 /* skip all other gathers */
 211                 if (cmdbuf != reloc->cmdbuf.bo)
 212                         continue;
 213
 214                 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) {
 215                         target = (u32 *)job->gather_copy_mapped +
 216                                         reloc->cmdbuf.offset / sizeof(u32) +
 217                                                 g->offset / sizeof(u32);
 218                         goto patch_reloc;
 219                 }
 220
 221                 if (last_page != reloc->cmdbuf.offset >> PAGE_SHIFT) {
 222                         if (cmdbuf_page_addr)
 223                                 host1x_bo_kunmap(cmdbuf, last_page,
 224                                                  cmdbuf_page_addr);
 225
 226                         cmdbuf_page_addr = host1x_bo_kmap(cmdbuf,
 227                                         reloc->cmdbuf.offset >> PAGE_SHIFT);
 228                         last_page = reloc->cmdbuf.offset >> PAGE_SHIFT;
 229
 230                         if (unlikely(!cmdbuf_page_addr)) {
 231                                 pr_err("Could not map cmdbuf for relocation\n");
 232                                 return -ENOMEM;
 233                         }
 234                 }
 235
 236                 target = cmdbuf_page_addr + (reloc->cmdbuf.offset & ~PAGE_MASK);
 237 patch_reloc:
 238                 *target = reloc_addr;
 239         }
 240
 241         if (cmdbuf_page_addr)
 242                 host1x_bo_kunmap(cmdbuf, last_page, cmdbuf_page_addr);
 243
 244         return 0;
 245 }
 246
 247 static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf,
 248                         unsigned int offset)
 249 {
 250         offset *= sizeof(u32);
 251
 252         if (reloc->cmdbuf.bo != cmdbuf || reloc->cmdbuf.offset != offset)
 253                 return false;
 254
 255         /* relocation shift value validation isn't implemented yet */
 256         if (reloc->shift)
 257                 return false;
 258
 259         return true;
 260 }
 261
 262 struct host1x_firewall {
 263         struct host1x_job *job;
 264         struct device *dev;
 265
 266         unsigned int num_relocs;
 267         struct host1x_reloc *reloc;
 268
 269         struct host1x_bo *cmdbuf;
 270         unsigned int offset;
 271
 272         u32 words;
 273         u32 class;
 274         u32 reg;
 275         u32 mask;
 276         u32 count;
 277 };
 278
 279 static int check_register(struct host1x_firewall *fw, unsigned long offset)
 280 {
 281         if (!fw->job->is_addr_reg)
 282                 return 0;
 283
 284         if (fw->job->is_addr_reg(fw->dev, fw->class, offset)) {
 285                 if (!fw->num_relocs)
 286                         return -EINVAL;
 287
 288                 if (!check_reloc(fw->reloc, fw->cmdbuf, fw->offset))
 289                         return -EINVAL;
 290
 291                 fw->num_relocs--;
 292                 fw->reloc++;
 293         }
 294
 295         return 0;
 296 }
 297
 298 static int check_class(struct host1x_firewall *fw, u32 class)
 299 {
 300         if (!fw->job->is_valid_class) {
 301                 if (fw->class != class)
 302                         return -EINVAL;
 303         } else {
 304                 if (!fw->job->is_valid_class(fw->class))
 305                         return -EINVAL;
 306         }
 307
 308         return 0;
 309 }
 310
 311 static int check_mask(struct host1x_firewall *fw)
 312 {
 313         u32 mask = fw->mask;
 314         u32 reg = fw->reg;
 315         int ret;
 316
 317         while (mask) {
 318                 if (fw->words == 0)
 319                         return -EINVAL;
 320
 321                 if (mask & 1) {
 322                         ret = check_register(fw, reg);
 323                         if (ret < 0)
 324                                 return ret;
 325
 326                         fw->words--;
 327                         fw->offset++;
 328                 }
 329                 mask >>= 1;
 330                 reg++;
 331         }
 332
 333         return 0;
 334 }
 335
 336 static int check_incr(struct host1x_firewall *fw)
 337 {
 338         u32 count = fw->count;
 339         u32 reg = fw->reg;
 340         int ret;
 341
 342         while (count) {
 343                 if (fw->words == 0)
 344                         return -EINVAL;
 345
 346                 ret = check_register(fw, reg);
 347                 if (ret < 0)
 348                         return ret;
 349
 350                 reg++;
 351                 fw->words--;
 352                 fw->offset++;
 353                 count--;
 354         }
 355
 356         return 0;
 357 }
 358
 359 static int check_nonincr(struct host1x_firewall *fw)
 360 {
 361         u32 count = fw->count;
 362         int ret;
 363
 364         while (count) {
 365                 if (fw->words == 0)
 366                         return -EINVAL;
 367
 368                 ret = check_register(fw, fw->reg);
 369                 if (ret < 0)
 370                         return ret;
 371
 372                 fw->words--;
 373                 fw->offset++;
 374                 count--;
 375         }
 376
 377         return 0;
 378 }
 379
 380 static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g)
 381 {
 382         u32 *cmdbuf_base = (u32 *)fw->job->gather_copy_mapped +
 383                 (g->offset / sizeof(u32));
 384         u32 job_class = fw->class;
 385         int err = 0;
 386
 387         fw->words = g->words;
 388         fw->cmdbuf = g->bo;
 389         fw->offset = 0;
 390
 391         while (fw->words && !err) {
 392                 u32 word = cmdbuf_base[fw->offset];
 393                 u32 opcode = (word & 0xf0000000) >> 28;
 394
 395                 fw->mask = 0;
 396                 fw->reg = 0;
 397                 fw->count = 0;
 398                 fw->words--;
 399                 fw->offset++;
 400
 401                 switch (opcode) {
 402                 case 0:
 403                         fw->class = word >> 6 & 0x3ff;
 404                         fw->mask = word & 0x3f;
 405                         fw->reg = word >> 16 & 0xfff;
 406                         err = check_class(fw, job_class);
 407                         if (!err)
 408                                 err = check_mask(fw);
 409                         if (err)
 410                                 goto out;
 411                         break;
 412                 case 1:
 413                         fw->reg = word >> 16 & 0xfff;
 414                         fw->count = word & 0xffff;
 415                         err = check_incr(fw);
 416                         if (err)
 417                                 goto out;
 418                         break;
 419
 420                 case 2:
 421                         fw->reg = word >> 16 & 0xfff;
 422                         fw->count = word & 0xffff;
 423                         err = check_nonincr(fw);
 424                         if (err)
 425                                 goto out;
 426                         break;
 427
 428                 case 3:
 429                         fw->mask = word & 0xffff;
 430                         fw->reg = word >> 16 & 0xfff;
 431                         err = check_mask(fw);
 432                         if (err)
 433                                 goto out;
 434                         break;
 435                 case 4:
 436                 case 14:
 437                         break;
 438                 default:
 439                         err = -EINVAL;
 440                         break;
 441                 }
 442         }
 443
 444 out:
 445         return err;
 446 }
 447
 448 static inline int copy_gathers(struct host1x_job *job, struct device *dev)
 449 {
 450         struct host1x_firewall fw;
 451         size_t size = 0;
 452         size_t offset = 0;
 453         unsigned int i;
 454
 455         fw.job = job;
 456         fw.dev = dev;
 457         fw.reloc = job->relocs;
 458         fw.num_relocs = job->num_relocs;
 459         fw.class = job->class;
 460
 461         for (i = 0; i < job->num_gathers; i++) {
 462                 struct host1x_job_gather *g = &job->gathers[i];
 463
 464                 size += g->words * sizeof(u32);
 465         }
 466
 467         /*
 468          * Try a non-blocking allocation from a higher priority pools first,
 469          * as awaiting for the allocation here is a major performance hit.
 470          */
 471         job->gather_copy_mapped = dma_alloc_wc(dev, size, &job->gather_copy,
 472                                                GFP_NOWAIT);
 473
 474         /* the higher priority allocation failed, try the generic-blocking */
 475         if (!job->gather_copy_mapped)
 476                 job->gather_copy_mapped = dma_alloc_wc(dev, size,
 477                                                        &job->gather_copy,
 478                                                        GFP_KERNEL);
 479         if (!job->gather_copy_mapped)
 480                 return -ENOMEM;
 481
 482         job->gather_copy_size = size;
 483
 484         for (i = 0; i < job->num_gathers; i++) {
 485                 struct host1x_job_gather *g = &job->gathers[i];
 486                 void *gather;
 487
 488                 /* Copy the gather */
 489                 gather = host1x_bo_mmap(g->bo);
 490                 memcpy(job->gather_copy_mapped + offset, gather + g->offset,
 491                        g->words * sizeof(u32));
 492                 host1x_bo_munmap(g->bo, gather);
 493
 494                 /* Store the location in the buffer */
 495                 g->base = job->gather_copy;
 496                 g->offset = offset;
 497
 498                 /* Validate the job */
 499                 if (validate(&fw, g))
 500                         return -EINVAL;
 501
 502                 offset += g->words * sizeof(u32);
 503         }
 504
 505         /* No relocs should remain at this point */
 506         if (fw.num_relocs)
 507                 return -EINVAL;
 508
 509         return 0;
 510 }
 511
 512 int host1x_job_pin(struct host1x_job *job, struct device *dev)
 513 {
 514         int err;
 515         unsigned int i, j;
 516         struct host1x *host = dev_get_drvdata(dev->parent);
 517
 518         /* pin memory */
 519         err = pin_job(host, job);
 520         if (err)
 521                 goto out;
 522
 523         if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) {
 524                 err = copy_gathers(job, dev);
 525                 if (err)
 526                         goto out;
 527         }
 528
 529         /* patch gathers */
 530         for (i = 0; i < job->num_gathers; i++) {
 531                 struct host1x_job_gather *g = &job->gathers[i];
 532
 533                 /* process each gather mem only once */
 534                 if (g->handled)
 535                         continue;
 536
 537                 /* copy_gathers() sets gathers base if firewall is enabled */
 538                 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL))
 539                         g->base = job->gather_addr_phys[i];
 540
 541                 for (j = i + 1; j < job->num_gathers; j++) {
 542                         if (job->gathers[j].bo == g->bo) {
 543                                 job->gathers[j].handled = true;
 544                                 job->gathers[j].base = g->base;
 545                         }
 546                 }
 547
 548                 err = do_relocs(job, g);
 549                 if (err)
 550                         break;
 551         }
 552
 553 out:
 554         if (err)
 555                 host1x_job_unpin(job);
 556         wmb();
 557
 558         return err;
 559 }
 560 EXPORT_SYMBOL(host1x_job_pin);
 561
 562 void host1x_job_unpin(struct host1x_job *job)
 563 {
 564         struct host1x *host = dev_get_drvdata(job->channel->dev->parent);
 565         unsigned int i;
 566
 567         for (i = 0; i < job->num_unpins; i++) {
 568                 struct host1x_job_unpin_data *unpin = &job->unpins[i];
 569
 570                 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) &&
 571                     unpin->size && host->domain) {
 572                         iommu_unmap(host->domain, job->addr_phys[i],
 573                                     unpin->size);
 574                         free_iova(&host->iova,
 575                                 iova_pfn(&host->iova, job->addr_phys[i]));
 576                 }
 577
 578                 host1x_bo_unpin(host->dev, unpin->bo, unpin->sgt);
 579                 host1x_bo_put(unpin->bo);
 580         }
 581
 582         job->num_unpins = 0;
 583
 584         if (job->gather_copy_size)
 585                 dma_free_wc(job->channel->dev, job->gather_copy_size,
 586                             job->gather_copy_mapped, job->gather_copy);
 587 }
 588 EXPORT_SYMBOL(host1x_job_unpin);
 589
 590 /*
 591  * Debug routine used to dump job entries
 592  */
 593 void host1x_job_dump(struct device *dev, struct host1x_job *job)
 594 {
 595         dev_dbg(dev, "    SYNCPT_ID   %d\n", job->syncpt_id);
 596         dev_dbg(dev, "    SYNCPT_VAL  %d\n", job->syncpt_end);
 597         dev_dbg(dev, "    FIRST_GET   0x%x\n", job->first_get);
 598         dev_dbg(dev, "    TIMEOUT     %d\n", job->timeout);
 599         dev_dbg(dev, "    NUM_SLOTS   %d\n", job->num_slots);
 600         dev_dbg(dev, "    NUM_HANDLES %d\n", job->num_unpins);
 601 }