drivers/misc/habanalabs/hw_queue.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include "habanalabs.h"
   9
  10 #include <linux/slab.h>
  11
  12 /*
  13  * hl_queue_add_ptr - add to pi or ci and checks if it wraps around
  14  *
  15  * @ptr: the current pi/ci value
  16  * @val: the amount to add
  17  *
  18  * Add val to ptr. It can go until twice the queue length.
  19  */
  20 inline u32 hl_hw_queue_add_ptr(u32 ptr, u16 val)
  21 {
  22         ptr += val;
  23         ptr &= ((HL_QUEUE_LENGTH << 1) - 1);
  24         return ptr;
  25 }
  26
  27 static inline int queue_free_slots(struct hl_hw_queue *q, u32 queue_len)
  28 {
  29         int delta = (q->pi - q->ci);
  30
  31         if (delta >= 0)
  32                 return (queue_len - delta);
  33         else
  34                 return (abs(delta) - queue_len);
  35 }
  36
  37 void hl_int_hw_queue_update_ci(struct hl_cs *cs)
  38 {
  39         struct hl_device *hdev = cs->ctx->hdev;
  40         struct hl_hw_queue *q;
  41         int i;
  42
  43         hdev->asic_funcs->hw_queues_lock(hdev);
  44
  45         if (hdev->disabled)
  46                 goto out;
  47
  48         q = &hdev->kernel_queues[0];
  49         for (i = 0 ; i < HL_MAX_QUEUES ; i++, q++) {
  50                 if (q->queue_type == QUEUE_TYPE_INT) {
  51                         q->ci += cs->jobs_in_queue_cnt[i];
  52                         q->ci &= ((q->int_queue_len << 1) - 1);
  53                 }
  54         }
  55
  56 out:
  57         hdev->asic_funcs->hw_queues_unlock(hdev);
  58 }
  59
  60 /*
  61  * ext_queue_submit_bd - Submit a buffer descriptor to an external queue
  62  *
  63  * @hdev: pointer to habanalabs device structure
  64  * @q: pointer to habanalabs queue structure
  65  * @ctl: BD's control word
  66  * @len: BD's length
  67  * @ptr: BD's pointer
  68  *
  69  * This function assumes there is enough space on the queue to submit a new
  70  * BD to it. It initializes the next BD and calls the device specific
  71  * function to set the pi (and doorbell)
  72  *
  73  * This function must be called when the scheduler mutex is taken
  74  *
  75  */
  76 static void ext_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q,
  77                                 u32 ctl, u32 len, u64 ptr)
  78 {
  79         struct hl_bd *bd;
  80
  81         bd = (struct hl_bd *) (uintptr_t) q->kernel_address;
  82         bd += hl_pi_2_offset(q->pi);
  83         bd->ctl = __cpu_to_le32(ctl);
  84         bd->len = __cpu_to_le32(len);
  85         bd->ptr = __cpu_to_le64(ptr);
  86
  87         q->pi = hl_queue_inc_ptr(q->pi);
  88         hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);
  89 }
  90
  91 /*
  92  * ext_queue_sanity_checks - perform some sanity checks on external queue
  93  *
  94  * @hdev              : pointer to hl_device structure
  95  * @q                 : pointer to hl_hw_queue structure
  96  * @num_of_entries    : how many entries to check for space
  97  * @reserve_cq_entry  : whether to reserve an entry in the cq
  98  *
  99  * H/W queues spinlock should be taken before calling this function
 100  *
 101  * Perform the following:
 102  * - Make sure we have enough space in the h/w queue
 103  * - Make sure we have enough space in the completion queue
 104  * - Reserve space in the completion queue (needs to be reversed if there
 105  *   is a failure down the road before the actual submission of work). Only
 106  *   do this action if reserve_cq_entry is true
 107  *
 108  */
 109 static int ext_queue_sanity_checks(struct hl_device *hdev,
 110                                 struct hl_hw_queue *q, int num_of_entries,
 111                                 bool reserve_cq_entry)
 112 {
 113         atomic_t *free_slots =
 114                         &hdev->completion_queue[q->hw_queue_id].free_slots_cnt;
 115         int free_slots_cnt;
 116
 117         /* Check we have enough space in the queue */
 118         free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH);
 119
 120         if (free_slots_cnt < num_of_entries) {
 121                 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",
 122                         q->hw_queue_id, num_of_entries);
 123                 return -EAGAIN;
 124         }
 125
 126         if (reserve_cq_entry) {
 127                 /*
 128                  * Check we have enough space in the completion queue
 129                  * Add -1 to counter (decrement) unless counter was already 0
 130                  * In that case, CQ is full so we can't submit a new CB because
 131                  * we won't get ack on its completion
 132                  * atomic_add_unless will return 0 if counter was already 0
 133                  */
 134                 if (atomic_add_negative(num_of_entries * -1, free_slots)) {
 135                         dev_dbg(hdev->dev, "No space for %d on CQ %d\n",
 136                                 num_of_entries, q->hw_queue_id);
 137                         atomic_add(num_of_entries, free_slots);
 138                         return -EAGAIN;
 139                 }
 140         }
 141
 142         return 0;
 143 }
 144
 145 /*
 146  * int_queue_sanity_checks - perform some sanity checks on internal queue
 147  *
 148  * @hdev              : pointer to hl_device structure
 149  * @q                 : pointer to hl_hw_queue structure
 150  * @num_of_entries    : how many entries to check for space
 151  *
 152  * H/W queues spinlock should be taken before calling this function
 153  *
 154  * Perform the following:
 155  * - Make sure we have enough space in the h/w queue
 156  *
 157  */
 158 static int int_queue_sanity_checks(struct hl_device *hdev,
 159                                         struct hl_hw_queue *q,
 160                                         int num_of_entries)
 161 {
 162         int free_slots_cnt;
 163
 164         /* Check we have enough space in the queue */
 165         free_slots_cnt = queue_free_slots(q, q->int_queue_len);
 166
 167         if (free_slots_cnt < num_of_entries) {
 168                 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",
 169                         q->hw_queue_id, num_of_entries);
 170                 return -EAGAIN;
 171         }
 172
 173         return 0;
 174 }
 175
 176 /*
 177  * hl_hw_queue_send_cb_no_cmpl - send a single CB (not a JOB) without completion
 178  *
 179  * @hdev: pointer to hl_device structure
 180  * @hw_queue_id: Queue's type
 181  * @cb_size: size of CB
 182  * @cb_ptr: pointer to CB location
 183  *
 184  * This function sends a single CB, that must NOT generate a completion entry
 185  *
 186  */
 187 int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
 188                                 u32 cb_size, u64 cb_ptr)
 189 {
 190         struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];
 191         int rc;
 192
 193         /*
 194          * The CPU queue is a synchronous queue with an effective depth of
 195          * a single entry (although it is allocated with room for multiple
 196          * entries). Therefore, there is a different lock, called
 197          * send_cpu_message_lock, that serializes accesses to the CPU queue.
 198          * As a result, we don't need to lock the access to the entire H/W
 199          * queues module when submitting a JOB to the CPU queue
 200          */
 201         if (q->queue_type != QUEUE_TYPE_CPU)
 202                 hdev->asic_funcs->hw_queues_lock(hdev);
 203
 204         if (hdev->disabled) {
 205                 rc = -EPERM;
 206                 goto out;
 207         }
 208
 209         rc = ext_queue_sanity_checks(hdev, q, 1, false);
 210         if (rc)
 211                 goto out;
 212
 213         ext_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);
 214
 215 out:
 216         if (q->queue_type != QUEUE_TYPE_CPU)
 217                 hdev->asic_funcs->hw_queues_unlock(hdev);
 218
 219         return rc;
 220 }
 221
 222 /*
 223  * ext_hw_queue_schedule_job - submit an JOB to an external queue
 224  *
 225  * @job: pointer to the job that needs to be submitted to the queue
 226  *
 227  * This function must be called when the scheduler mutex is taken
 228  *
 229  */
 230 static void ext_hw_queue_schedule_job(struct hl_cs_job *job)
 231 {
 232         struct hl_device *hdev = job->cs->ctx->hdev;
 233         struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];
 234         struct hl_cq_entry cq_pkt;
 235         struct hl_cq *cq;
 236         u64 cq_addr;
 237         struct hl_cb *cb;
 238         u32 ctl;
 239         u32 len;
 240         u64 ptr;
 241
 242         /*
 243          * Update the JOB ID inside the BD CTL so the device would know what
 244          * to write in the completion queue
 245          */
 246         ctl = ((q->pi << BD_CTL_SHADOW_INDEX_SHIFT) & BD_CTL_SHADOW_INDEX_MASK);
 247
 248         cb = job->patched_cb;
 249         len = job->job_cb_size;
 250         ptr = cb->bus_address;
 251
 252         cq_pkt.data = __cpu_to_le32(
 253                                 ((q->pi << CQ_ENTRY_SHADOW_INDEX_SHIFT)
 254                                         & CQ_ENTRY_SHADOW_INDEX_MASK) |
 255                                 (1 << CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT) |
 256                                 (1 << CQ_ENTRY_READY_SHIFT));
 257
 258         /*
 259          * No need to protect pi_offset because scheduling to the
 260          * H/W queues is done under the scheduler mutex
 261          *
 262          * No need to check if CQ is full because it was already
 263          * checked in hl_queue_sanity_checks
 264          */
 265         cq = &hdev->completion_queue[q->hw_queue_id];
 266         cq_addr = cq->bus_address + cq->pi * sizeof(struct hl_cq_entry);
 267
 268         hdev->asic_funcs->add_end_of_cb_packets(cb->kernel_address, len,
 269                                                 cq_addr,
 270                                                 __le32_to_cpu(cq_pkt.data),
 271                                                 q->hw_queue_id);
 272
 273         q->shadow_queue[hl_pi_2_offset(q->pi)] = job;
 274
 275         cq->pi = hl_cq_inc_ptr(cq->pi);
 276
 277         ext_queue_submit_bd(hdev, q, ctl, len, ptr);
 278 }
 279
 280 /*
 281  * int_hw_queue_schedule_job - submit an JOB to an internal queue
 282  *
 283  * @job: pointer to the job that needs to be submitted to the queue
 284  *
 285  * This function must be called when the scheduler mutex is taken
 286  *
 287  */
 288 static void int_hw_queue_schedule_job(struct hl_cs_job *job)
 289 {
 290         struct hl_device *hdev = job->cs->ctx->hdev;
 291         struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];
 292         struct hl_bd bd;
 293         u64 *pi, *pbd = (u64 *) &bd;
 294
 295         bd.ctl = 0;
 296         bd.len = __cpu_to_le32(job->job_cb_size);
 297         bd.ptr = __cpu_to_le64((u64) (uintptr_t) job->user_cb);
 298
 299         pi = (u64 *) (uintptr_t) (q->kernel_address +
 300                 ((q->pi & (q->int_queue_len - 1)) * sizeof(bd)));
 301
 302         pi[0] = pbd[0];
 303         pi[1] = pbd[1];
 304
 305         q->pi++;
 306         q->pi &= ((q->int_queue_len << 1) - 1);
 307
 308         /* Flush PQ entry write. Relevant only for specific ASICs */
 309         hdev->asic_funcs->flush_pq_write(hdev, pi, pbd[0]);
 310
 311         hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);
 312 }
 313
 314 /*
 315  * hl_hw_queue_schedule_cs - schedule a command submission
 316  *
 317  * @job        : pointer to the CS
 318  *
 319  */
 320 int hl_hw_queue_schedule_cs(struct hl_cs *cs)
 321 {
 322         struct hl_device *hdev = cs->ctx->hdev;
 323         struct hl_cs_job *job, *tmp;
 324         struct hl_hw_queue *q;
 325         int rc = 0, i, cq_cnt;
 326
 327         hdev->asic_funcs->hw_queues_lock(hdev);
 328
 329         if (hl_device_disabled_or_in_reset(hdev)) {
 330                 dev_err(hdev->dev,
 331                         "device is disabled or in reset, CS rejected!\n");
 332                 rc = -EPERM;
 333                 goto out;
 334         }
 335
 336         q = &hdev->kernel_queues[0];
 337         /* This loop assumes all external queues are consecutive */
 338         for (i = 0, cq_cnt = 0 ; i < HL_MAX_QUEUES ; i++, q++) {
 339                 if (q->queue_type == QUEUE_TYPE_EXT) {
 340                         if (cs->jobs_in_queue_cnt[i]) {
 341                                 rc = ext_queue_sanity_checks(hdev, q,
 342                                         cs->jobs_in_queue_cnt[i], true);
 343                                 if (rc)
 344                                         goto unroll_cq_resv;
 345                                 cq_cnt++;
 346                         }
 347                 } else if (q->queue_type == QUEUE_TYPE_INT) {
 348                         if (cs->jobs_in_queue_cnt[i]) {
 349                                 rc = int_queue_sanity_checks(hdev, q,
 350                                         cs->jobs_in_queue_cnt[i]);
 351                                 if (rc)
 352                                         goto unroll_cq_resv;
 353                         }
 354                 }
 355         }
 356
 357         spin_lock(&hdev->hw_queues_mirror_lock);
 358         list_add_tail(&cs->mirror_node, &hdev->hw_queues_mirror_list);
 359
 360         /* Queue TDR if the CS is the first entry and if timeout is wanted */
 361         if ((hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) &&
 362                         (list_first_entry(&hdev->hw_queues_mirror_list,
 363                                         struct hl_cs, mirror_node) == cs)) {
 364                 cs->tdr_active = true;
 365                 schedule_delayed_work(&cs->work_tdr, hdev->timeout_jiffies);
 366                 spin_unlock(&hdev->hw_queues_mirror_lock);
 367         } else {
 368                 spin_unlock(&hdev->hw_queues_mirror_lock);
 369         }
 370
 371         atomic_inc(&hdev->cs_active_cnt);
 372
 373         list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)
 374                 if (job->ext_queue)
 375                         ext_hw_queue_schedule_job(job);
 376                 else
 377                         int_hw_queue_schedule_job(job);
 378
 379         cs->submitted = true;
 380
 381         goto out;
 382
 383 unroll_cq_resv:
 384         /* This loop assumes all external queues are consecutive */
 385         q = &hdev->kernel_queues[0];
 386         for (i = 0 ; (i < HL_MAX_QUEUES) && (cq_cnt > 0) ; i++, q++) {
 387                 if ((q->queue_type == QUEUE_TYPE_EXT) &&
 388                                 (cs->jobs_in_queue_cnt[i])) {
 389                         atomic_t *free_slots =
 390                                 &hdev->completion_queue[i].free_slots_cnt;
 391                         atomic_add(cs->jobs_in_queue_cnt[i], free_slots);
 392                         cq_cnt--;
 393                 }
 394         }
 395
 396 out:
 397         hdev->asic_funcs->hw_queues_unlock(hdev);
 398
 399         return rc;
 400 }
 401
 402 /*
 403  * hl_hw_queue_inc_ci_kernel - increment ci for kernel's queue
 404  *
 405  * @hdev: pointer to hl_device structure
 406  * @hw_queue_id: which queue to increment its ci
 407  */
 408 void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id)
 409 {
 410         struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];
 411
 412         q->ci = hl_queue_inc_ptr(q->ci);
 413 }
 414
 415 static int ext_and_cpu_hw_queue_init(struct hl_device *hdev,
 416                                 struct hl_hw_queue *q, bool is_cpu_queue)
 417 {
 418         void *p;
 419         int rc;
 420
 421         if (is_cpu_queue)
 422                 p = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
 423                                                         HL_QUEUE_SIZE_IN_BYTES,
 424                                                         &q->bus_address);
 425         else
 426                 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
 427                                                 HL_QUEUE_SIZE_IN_BYTES,
 428                                                 &q->bus_address,
 429                                                 GFP_KERNEL | __GFP_ZERO);
 430         if (!p)
 431                 return -ENOMEM;
 432
 433         q->kernel_address = (u64) (uintptr_t) p;
 434
 435         q->shadow_queue = kmalloc_array(HL_QUEUE_LENGTH,
 436                                         sizeof(*q->shadow_queue),
 437                                         GFP_KERNEL);
 438         if (!q->shadow_queue) {
 439                 dev_err(hdev->dev,
 440                         "Failed to allocate shadow queue for H/W queue %d\n",
 441                         q->hw_queue_id);
 442                 rc = -ENOMEM;
 443                 goto free_queue;
 444         }
 445
 446         /* Make sure read/write pointers are initialized to start of queue */
 447         q->ci = 0;
 448         q->pi = 0;
 449
 450         return 0;
 451
 452 free_queue:
 453         if (is_cpu_queue)
 454                 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
 455                                         HL_QUEUE_SIZE_IN_BYTES,
 456                                         (void *) (uintptr_t) q->kernel_address);
 457         else
 458                 hdev->asic_funcs->asic_dma_free_coherent(hdev,
 459                                         HL_QUEUE_SIZE_IN_BYTES,
 460                                         (void *) (uintptr_t) q->kernel_address,
 461                                         q->bus_address);
 462
 463         return rc;
 464 }
 465
 466 static int int_hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
 467 {
 468         void *p;
 469
 470         p = hdev->asic_funcs->get_int_queue_base(hdev, q->hw_queue_id,
 471                                         &q->bus_address, &q->int_queue_len);
 472         if (!p) {
 473                 dev_err(hdev->dev,
 474                         "Failed to get base address for internal queue %d\n",
 475                         q->hw_queue_id);
 476                 return -EFAULT;
 477         }
 478
 479         q->kernel_address = (u64) (uintptr_t) p;
 480         q->pi = 0;
 481         q->ci = 0;
 482
 483         return 0;
 484 }
 485
 486 static int cpu_hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
 487 {
 488         return ext_and_cpu_hw_queue_init(hdev, q, true);
 489 }
 490
 491 static int ext_hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
 492 {
 493         return ext_and_cpu_hw_queue_init(hdev, q, false);
 494 }
 495
 496 /*
 497  * hw_queue_init - main initialization function for H/W queue object
 498  *
 499  * @hdev: pointer to hl_device device structure
 500  * @q: pointer to hl_hw_queue queue structure
 501  * @hw_queue_id: The id of the H/W queue
 502  *
 503  * Allocate dma-able memory for the queue and initialize fields
 504  * Returns 0 on success
 505  */
 506 static int hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q,
 507                         u32 hw_queue_id)
 508 {
 509         int rc;
 510
 511         BUILD_BUG_ON(HL_QUEUE_SIZE_IN_BYTES > HL_PAGE_SIZE);
 512
 513         q->hw_queue_id = hw_queue_id;
 514
 515         switch (q->queue_type) {
 516         case QUEUE_TYPE_EXT:
 517                 rc = ext_hw_queue_init(hdev, q);
 518                 break;
 519
 520         case QUEUE_TYPE_INT:
 521                 rc = int_hw_queue_init(hdev, q);
 522                 break;
 523
 524         case QUEUE_TYPE_CPU:
 525                 rc = cpu_hw_queue_init(hdev, q);
 526                 break;
 527
 528         case QUEUE_TYPE_NA:
 529                 q->valid = 0;
 530                 return 0;
 531
 532         default:
 533                 dev_crit(hdev->dev, "wrong queue type %d during init\n",
 534                         q->queue_type);
 535                 rc = -EINVAL;
 536                 break;
 537         }
 538
 539         if (rc)
 540                 return rc;
 541
 542         q->valid = 1;
 543
 544         return 0;
 545 }
 546
 547 /*
 548  * hw_queue_fini - destroy queue
 549  *
 550  * @hdev: pointer to hl_device device structure
 551  * @q: pointer to hl_hw_queue queue structure
 552  *
 553  * Free the queue memory
 554  */
 555 static void hw_queue_fini(struct hl_device *hdev, struct hl_hw_queue *q)
 556 {
 557         if (!q->valid)
 558                 return;
 559
 560         /*
 561          * If we arrived here, there are no jobs waiting on this queue
 562          * so we can safely remove it.
 563          * This is because this function can only called when:
 564          * 1. Either a context is deleted, which only can occur if all its
 565          *    jobs were finished
 566          * 2. A context wasn't able to be created due to failure or timeout,
 567          *    which means there are no jobs on the queue yet
 568          *
 569          * The only exception are the queues of the kernel context, but
 570          * if they are being destroyed, it means that the entire module is
 571          * being removed. If the module is removed, it means there is no open
 572          * user context. It also means that if a job was submitted by
 573          * the kernel driver (e.g. context creation), the job itself was
 574          * released by the kernel driver when a timeout occurred on its
 575          * Completion. Thus, we don't need to release it again.
 576          */
 577
 578         if (q->queue_type == QUEUE_TYPE_INT)
 579                 return;
 580
 581         kfree(q->shadow_queue);
 582
 583         if (q->queue_type == QUEUE_TYPE_CPU)
 584                 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
 585                                         HL_QUEUE_SIZE_IN_BYTES,
 586                                         (void *) (uintptr_t) q->kernel_address);
 587         else
 588                 hdev->asic_funcs->asic_dma_free_coherent(hdev,
 589                                         HL_QUEUE_SIZE_IN_BYTES,
 590                                         (void *) (uintptr_t) q->kernel_address,
 591                                         q->bus_address);
 592 }
 593
 594 int hl_hw_queues_create(struct hl_device *hdev)
 595 {
 596         struct asic_fixed_properties *asic = &hdev->asic_prop;
 597         struct hl_hw_queue *q;
 598         int i, rc, q_ready_cnt;
 599
 600         hdev->kernel_queues = kcalloc(HL_MAX_QUEUES,
 601                                 sizeof(*hdev->kernel_queues), GFP_KERNEL);
 602
 603         if (!hdev->kernel_queues) {
 604                 dev_err(hdev->dev, "Not enough memory for H/W queues\n");
 605                 return -ENOMEM;
 606         }
 607
 608         /* Initialize the H/W queues */
 609         for (i = 0, q_ready_cnt = 0, q = hdev->kernel_queues;
 610                         i < HL_MAX_QUEUES ; i++, q_ready_cnt++, q++) {
 611
 612                 q->queue_type = asic->hw_queues_props[i].type;
 613                 rc = hw_queue_init(hdev, q, i);
 614                 if (rc) {
 615                         dev_err(hdev->dev,
 616                                 "failed to initialize queue %d\n", i);
 617                         goto release_queues;
 618                 }
 619         }
 620
 621         return 0;
 622
 623 release_queues:
 624         for (i = 0, q = hdev->kernel_queues ; i < q_ready_cnt ; i++, q++)
 625                 hw_queue_fini(hdev, q);
 626
 627         kfree(hdev->kernel_queues);
 628
 629         return rc;
 630 }
 631
 632 void hl_hw_queues_destroy(struct hl_device *hdev)
 633 {
 634         struct hl_hw_queue *q;
 635         int i;
 636
 637         for (i = 0, q = hdev->kernel_queues ; i < HL_MAX_QUEUES ; i++, q++)
 638                 hw_queue_fini(hdev, q);
 639
 640         kfree(hdev->kernel_queues);
 641 }
 642
 643 void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset)
 644 {
 645         struct hl_hw_queue *q;
 646         int i;
 647
 648         for (i = 0, q = hdev->kernel_queues ; i < HL_MAX_QUEUES ; i++, q++) {
 649                 if ((!q->valid) ||
 650                         ((!hard_reset) && (q->queue_type == QUEUE_TYPE_CPU)))
 651                         continue;
 652                 q->pi = q->ci = 0;
 653         }
 654 }