1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #define pr_fmt(fmt) "habanalabs: " fmt
10 #include "habanalabs.h"
12 #include <linux/pci.h>
13 #include <linux/sched/signal.h>
14 #include <linux/hwmon.h>
15 #include <uapi/misc/habanalabs.h>
17 #define HL_PLDM_PENDING_RESET_PER_SEC (HL_PENDING_RESET_PER_SEC * 10)
19 bool hl_device_disabled_or_in_reset(struct hl_device *hdev)
21 if ((hdev->disabled) || (atomic_read(&hdev->in_reset)))
27 enum hl_device_status hl_device_status(struct hl_device *hdev)
29 enum hl_device_status status;
32 status = HL_DEVICE_STATUS_MALFUNCTION;
33 else if (atomic_read(&hdev->in_reset))
34 status = HL_DEVICE_STATUS_IN_RESET;
36 status = HL_DEVICE_STATUS_OPERATIONAL;
41 static void hpriv_release(struct kref *ref)
43 struct hl_fpriv *hpriv;
44 struct hl_device *hdev;
46 hpriv = container_of(ref, struct hl_fpriv, refcount);
50 put_pid(hpriv->taskpid);
52 hl_debugfs_remove_file(hpriv);
54 mutex_destroy(&hpriv->restore_phase_mutex);
58 /* Now the FD is really closed */
59 atomic_dec(&hdev->fd_open_cnt);
61 /* This allows a new user context to open the device */
62 hdev->user_ctx = NULL;
65 void hl_hpriv_get(struct hl_fpriv *hpriv)
67 kref_get(&hpriv->refcount);
70 void hl_hpriv_put(struct hl_fpriv *hpriv)
72 kref_put(&hpriv->refcount, hpriv_release);
76 * hl_device_release - release function for habanalabs device
78 * @inode: pointer to inode structure
79 * @filp: pointer to file structure
81 * Called when process closes an habanalabs device
83 static int hl_device_release(struct inode *inode, struct file *filp)
85 struct hl_fpriv *hpriv = filp->private_data;
87 hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);
88 hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
90 filp->private_data = NULL;
98 * hl_mmap - mmap function for habanalabs device
100 * @*filp: pointer to file structure
101 * @*vma: pointer to vm_area_struct of the process
103 * Called when process does an mmap on habanalabs device. Call the device's mmap
104 * function at the end of the common code.
106 static int hl_mmap(struct file *filp, struct vm_area_struct *vma)
108 struct hl_fpriv *hpriv = filp->private_data;
110 if ((vma->vm_pgoff & HL_MMAP_CB_MASK) == HL_MMAP_CB_MASK) {
111 vma->vm_pgoff ^= HL_MMAP_CB_MASK;
112 return hl_cb_mmap(hpriv, vma);
118 static const struct file_operations hl_ops = {
119 .owner = THIS_MODULE,
120 .open = hl_device_open,
121 .release = hl_device_release,
123 .unlocked_ioctl = hl_ioctl,
124 .compat_ioctl = hl_ioctl
128 * device_setup_cdev - setup cdev and device for habanalabs device
130 * @hdev: pointer to habanalabs device structure
131 * @hclass: pointer to the class object of the device
132 * @minor: minor number of the specific device
133 * @fpos : file operations to install for this device
135 * Create a cdev and a Linux device for habanalabs's device. Need to be
136 * called at the end of the habanalabs device initialization process,
137 * because this function exposes the device to the user
139 static int device_setup_cdev(struct hl_device *hdev, struct class *hclass,
140 int minor, const struct file_operations *fops)
142 int err, devno = MKDEV(hdev->major, minor);
143 struct cdev *hdev_cdev = &hdev->cdev;
146 name = kasprintf(GFP_KERNEL, "hl%d", hdev->id);
150 cdev_init(hdev_cdev, fops);
151 hdev_cdev->owner = THIS_MODULE;
152 err = cdev_add(hdev_cdev, devno, 1);
154 pr_err("Failed to add char device %s\n", name);
158 hdev->dev = device_create(hclass, NULL, devno, NULL, "%s", name);
159 if (IS_ERR(hdev->dev)) {
160 pr_err("Failed to create device %s\n", name);
161 err = PTR_ERR(hdev->dev);
162 goto err_device_create;
165 dev_set_drvdata(hdev->dev, hdev);
179 * device_early_init - do some early initialization for the habanalabs device
181 * @hdev: pointer to habanalabs device structure
183 * Install the relevant function pointers and call the early_init function,
184 * if such a function exists
186 static int device_early_init(struct hl_device *hdev)
190 switch (hdev->asic_type) {
192 goya_set_asic_funcs(hdev);
193 strlcpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
196 dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
201 rc = hdev->asic_funcs->early_init(hdev);
205 rc = hl_asid_init(hdev);
209 hdev->cq_wq = alloc_workqueue("hl-free-jobs", WQ_UNBOUND, 0);
210 if (hdev->cq_wq == NULL) {
211 dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
216 hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0);
217 if (hdev->eq_wq == NULL) {
218 dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
223 hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),
225 if (!hdev->hl_chip_info) {
230 hl_cb_mgr_init(&hdev->kernel_cb_mgr);
232 mutex_init(&hdev->fd_open_cnt_lock);
233 mutex_init(&hdev->send_cpu_message_lock);
234 INIT_LIST_HEAD(&hdev->hw_queues_mirror_list);
235 spin_lock_init(&hdev->hw_queues_mirror_lock);
236 atomic_set(&hdev->in_reset, 0);
237 atomic_set(&hdev->fd_open_cnt, 0);
238 atomic_set(&hdev->cs_active_cnt, 0);
243 destroy_workqueue(hdev->eq_wq);
245 destroy_workqueue(hdev->cq_wq);
249 if (hdev->asic_funcs->early_fini)
250 hdev->asic_funcs->early_fini(hdev);
256 * device_early_fini - finalize all that was done in device_early_init
258 * @hdev: pointer to habanalabs device structure
261 static void device_early_fini(struct hl_device *hdev)
263 mutex_destroy(&hdev->send_cpu_message_lock);
265 hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
267 kfree(hdev->hl_chip_info);
269 destroy_workqueue(hdev->eq_wq);
270 destroy_workqueue(hdev->cq_wq);
274 if (hdev->asic_funcs->early_fini)
275 hdev->asic_funcs->early_fini(hdev);
277 mutex_destroy(&hdev->fd_open_cnt_lock);
280 static void set_freq_to_low_job(struct work_struct *work)
282 struct hl_device *hdev = container_of(work, struct hl_device,
285 if (atomic_read(&hdev->fd_open_cnt) == 0)
286 hl_device_set_frequency(hdev, PLL_LOW);
288 schedule_delayed_work(&hdev->work_freq,
289 usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
292 static void hl_device_heartbeat(struct work_struct *work)
294 struct hl_device *hdev = container_of(work, struct hl_device,
295 work_heartbeat.work);
297 if (hl_device_disabled_or_in_reset(hdev))
300 if (!hdev->asic_funcs->send_heartbeat(hdev))
303 dev_err(hdev->dev, "Device heartbeat failed!\n");
304 hl_device_reset(hdev, true, false);
309 schedule_delayed_work(&hdev->work_heartbeat,
310 usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
314 * device_late_init - do late stuff initialization for the habanalabs device
316 * @hdev: pointer to habanalabs device structure
318 * Do stuff that either needs the device H/W queues to be active or needs
319 * to happen after all the rest of the initialization is finished
321 static int device_late_init(struct hl_device *hdev)
325 INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job);
326 hdev->high_pll = hdev->asic_prop.high_pll;
328 /* force setting to low frequency */
329 atomic_set(&hdev->curr_pll_profile, PLL_LOW);
331 if (hdev->pm_mng_profile == PM_AUTO)
332 hdev->asic_funcs->set_pll_profile(hdev, PLL_LOW);
334 hdev->asic_funcs->set_pll_profile(hdev, PLL_LAST);
336 if (hdev->asic_funcs->late_init) {
337 rc = hdev->asic_funcs->late_init(hdev);
340 "failed late initialization for the H/W\n");
345 schedule_delayed_work(&hdev->work_freq,
346 usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
348 if (hdev->heartbeat) {
349 INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
350 schedule_delayed_work(&hdev->work_heartbeat,
351 usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
354 hdev->late_init_done = true;
360 * device_late_fini - finalize all that was done in device_late_init
362 * @hdev: pointer to habanalabs device structure
365 static void device_late_fini(struct hl_device *hdev)
367 if (!hdev->late_init_done)
370 cancel_delayed_work_sync(&hdev->work_freq);
372 cancel_delayed_work_sync(&hdev->work_heartbeat);
374 if (hdev->asic_funcs->late_fini)
375 hdev->asic_funcs->late_fini(hdev);
377 hdev->late_init_done = false;
381 * hl_device_set_frequency - set the frequency of the device
383 * @hdev: pointer to habanalabs device structure
384 * @freq: the new frequency value
386 * Change the frequency if needed.
387 * We allose to set PLL to low only if there is no user process
388 * Returns 0 if no change was done, otherwise returns 1;
390 int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq)
392 enum hl_pll_frequency old_freq =
393 (freq == PLL_HIGH) ? PLL_LOW : PLL_HIGH;
396 if (hdev->pm_mng_profile == PM_MANUAL)
399 ret = atomic_cmpxchg(&hdev->curr_pll_profile, old_freq, freq);
404 * in case we want to lower frequency, check if device is not
405 * opened. We must have a check here to workaround race condition with
408 if ((freq == PLL_LOW) && (atomic_read(&hdev->fd_open_cnt) > 0)) {
409 atomic_set(&hdev->curr_pll_profile, PLL_HIGH);
413 dev_dbg(hdev->dev, "Changing device frequency to %s\n",
414 freq == PLL_HIGH ? "high" : "low");
416 hdev->asic_funcs->set_pll_profile(hdev, freq);
422 * hl_device_suspend - initiate device suspend
424 * @hdev: pointer to habanalabs device structure
426 * Puts the hw in the suspend state (all asics).
427 * Returns 0 for success or an error on failure.
428 * Called at driver suspend.
430 int hl_device_suspend(struct hl_device *hdev)
434 pci_save_state(hdev->pdev);
436 /* Block future CS/VM/JOB completion operations */
437 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
439 dev_err(hdev->dev, "Can't suspend while in reset\n");
443 /* This blocks all other stuff that is not blocked by in_reset */
444 hdev->disabled = true;
447 * Flush anyone that is inside the critical section of enqueue
450 hdev->asic_funcs->hw_queues_lock(hdev);
451 hdev->asic_funcs->hw_queues_unlock(hdev);
453 /* Flush processes that are sending message to CPU */
454 mutex_lock(&hdev->send_cpu_message_lock);
455 mutex_unlock(&hdev->send_cpu_message_lock);
457 rc = hdev->asic_funcs->suspend(hdev);
460 "Failed to disable PCI access of device CPU\n");
462 /* Shut down the device */
463 pci_disable_device(hdev->pdev);
464 pci_set_power_state(hdev->pdev, PCI_D3hot);
470 * hl_device_resume - initiate device resume
472 * @hdev: pointer to habanalabs device structure
474 * Bring the hw back to operating state (all asics).
475 * Returns 0 for success or an error on failure.
476 * Called at driver resume.
478 int hl_device_resume(struct hl_device *hdev)
482 pci_set_power_state(hdev->pdev, PCI_D0);
483 pci_restore_state(hdev->pdev);
484 rc = pci_enable_device_mem(hdev->pdev);
487 "Failed to enable PCI device in resume\n");
491 pci_set_master(hdev->pdev);
493 rc = hdev->asic_funcs->resume(hdev);
495 dev_err(hdev->dev, "Failed to resume device after suspend\n");
500 hdev->disabled = false;
501 atomic_set(&hdev->in_reset, 0);
503 rc = hl_device_reset(hdev, true, false);
505 dev_err(hdev->dev, "Failed to reset device during resume\n");
512 pci_clear_master(hdev->pdev);
513 pci_disable_device(hdev->pdev);
518 static void device_kill_open_processes(struct hl_device *hdev)
520 u16 pending_total, pending_cnt;
521 struct task_struct *task = NULL;
524 pending_total = HL_PLDM_PENDING_RESET_PER_SEC;
526 pending_total = HL_PENDING_RESET_PER_SEC;
528 pending_cnt = pending_total;
530 /* Flush all processes that are inside hl_open */
531 mutex_lock(&hdev->fd_open_cnt_lock);
533 while ((atomic_read(&hdev->fd_open_cnt)) && (pending_cnt)) {
538 "Can't HARD reset, waiting for user to close FD\n");
542 if (atomic_read(&hdev->fd_open_cnt)) {
543 task = get_pid_task(hdev->user_ctx->hpriv->taskpid,
546 dev_info(hdev->dev, "Killing user processes\n");
547 send_sig(SIGKILL, task, 1);
550 put_task_struct(task);
554 /* We killed the open users, but because the driver cleans up after the
555 * user contexts are closed (e.g. mmu mappings), we need to wait again
556 * to make sure the cleaning phase is finished before continuing with
560 pending_cnt = pending_total;
562 while ((atomic_read(&hdev->fd_open_cnt)) && (pending_cnt)) {
569 if (atomic_read(&hdev->fd_open_cnt))
571 "Going to hard reset with open user contexts\n");
573 mutex_unlock(&hdev->fd_open_cnt_lock);
577 static void device_hard_reset_pending(struct work_struct *work)
579 struct hl_device_reset_work *device_reset_work =
580 container_of(work, struct hl_device_reset_work, reset_work);
581 struct hl_device *hdev = device_reset_work->hdev;
583 device_kill_open_processes(hdev);
585 hl_device_reset(hdev, true, true);
587 kfree(device_reset_work);
591 * hl_device_reset - reset the device
593 * @hdev: pointer to habanalabs device structure
594 * @hard_reset: should we do hard reset to all engines or just reset the
595 * compute/dma engines
597 * Block future CS and wait for pending CS to be enqueued
599 * Flush all completions
600 * Re-initialize all internal data structures
601 * Call ASIC H/W init, late_init
605 * Returns 0 for success or an error on failure.
607 int hl_device_reset(struct hl_device *hdev, bool hard_reset,
608 bool from_hard_reset_thread)
612 if (!hdev->init_done) {
614 "Can't reset before initialization is done\n");
619 * Prevent concurrency in this function - only one reset should be
620 * done at any given time. Only need to perform this if we didn't
621 * get from the dedicated hard reset thread
623 if (!from_hard_reset_thread) {
624 /* Block future CS/VM/JOB completion operations */
625 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
629 /* This also blocks future CS/VM/JOB completion operations */
630 hdev->disabled = true;
633 * Flush anyone that is inside the critical section of enqueue
636 hdev->asic_funcs->hw_queues_lock(hdev);
637 hdev->asic_funcs->hw_queues_unlock(hdev);
639 dev_err(hdev->dev, "Going to RESET device!\n");
643 if ((hard_reset) && (!from_hard_reset_thread)) {
644 struct hl_device_reset_work *device_reset_work;
646 hdev->hard_reset_pending = true;
650 "Reset action is NOT supported in simulator\n");
655 device_reset_work = kzalloc(sizeof(*device_reset_work),
657 if (!device_reset_work) {
663 * Because the reset function can't run from interrupt or
664 * from heartbeat work, we need to call the reset function
665 * from a dedicated work
667 INIT_WORK(&device_reset_work->reset_work,
668 device_hard_reset_pending);
669 device_reset_work->hdev = hdev;
670 schedule_work(&device_reset_work->reset_work);
676 device_late_fini(hdev);
679 * Now that the heartbeat thread is closed, flush processes
680 * which are sending messages to CPU
682 mutex_lock(&hdev->send_cpu_message_lock);
683 mutex_unlock(&hdev->send_cpu_message_lock);
687 * Halt the engines and disable interrupts so we won't get any more
688 * completions from H/W and we won't have any accesses from the
689 * H/W to the host machine
691 hdev->asic_funcs->halt_engines(hdev, hard_reset);
693 /* Go over all the queues, release all CS and their jobs */
694 hl_cs_rollback_all(hdev);
696 /* Release kernel context */
697 if ((hard_reset) && (hl_ctx_put(hdev->kernel_ctx) == 1))
698 hdev->kernel_ctx = NULL;
700 /* Reset the H/W. It will be in idle state after this returns */
701 hdev->asic_funcs->hw_fini(hdev, hard_reset);
705 hl_eq_reset(hdev, &hdev->event_queue);
708 /* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */
709 hl_hw_queue_reset(hdev, hard_reset);
710 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
711 hl_cq_reset(hdev, &hdev->completion_queue[i]);
713 /* Make sure the context switch phase will run again */
714 if (hdev->user_ctx) {
715 atomic_set(&hdev->user_ctx->thread_ctx_switch_token, 1);
716 hdev->user_ctx->thread_ctx_switch_wait_token = 0;
719 /* Finished tear-down, starting to re-initialize */
722 hdev->device_cpu_disabled = false;
723 hdev->hard_reset_pending = false;
725 if (hdev->kernel_ctx) {
727 "kernel ctx was alive during hard reset, something is terribly wrong\n");
732 /* Allocate the kernel context */
733 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
735 if (!hdev->kernel_ctx) {
740 hdev->user_ctx = NULL;
742 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
745 "failed to init kernel ctx in hard reset\n");
746 kfree(hdev->kernel_ctx);
747 hdev->kernel_ctx = NULL;
752 rc = hdev->asic_funcs->hw_init(hdev);
755 "failed to initialize the H/W after reset\n");
759 hdev->disabled = false;
761 /* Check that the communication with the device is working */
762 rc = hdev->asic_funcs->test_queues(hdev);
765 "Failed to detect if device is alive after reset\n");
770 rc = device_late_init(hdev);
773 "Failed late init after hard reset\n");
777 rc = hl_vm_init(hdev);
780 "Failed to init memory module after hard reset\n");
784 hl_set_max_power(hdev, hdev->max_power);
786 rc = hdev->asic_funcs->soft_reset_late_init(hdev);
789 "Failed late init after soft reset\n");
794 atomic_set(&hdev->in_reset, 0);
797 hdev->hard_reset_cnt++;
799 hdev->soft_reset_cnt++;
804 hdev->disabled = true;
808 "Failed to reset! Device is NOT usable\n");
809 hdev->hard_reset_cnt++;
812 "Failed to do soft-reset, trying hard reset\n");
813 hdev->soft_reset_cnt++;
818 atomic_set(&hdev->in_reset, 0);
824 * hl_device_init - main initialization function for habanalabs device
826 * @hdev: pointer to habanalabs device structure
828 * Allocate an id for the device, do early initialization and then call the
829 * ASIC specific initialization functions. Finally, create the cdev and the
830 * Linux device to expose it to the user
832 int hl_device_init(struct hl_device *hdev, struct class *hclass)
834 int i, rc, cq_ready_cnt;
837 rc = device_setup_cdev(hdev, hclass, hdev->id, &hl_ops);
842 /* Initialize ASIC function pointers and perform early init */
843 rc = device_early_init(hdev);
848 * Start calling ASIC initialization. First S/W then H/W and finally
851 rc = hdev->asic_funcs->sw_init(hdev);
856 * Initialize the H/W queues. Must be done before hw_init, because
857 * there the addresses of the kernel queue are being written to the
858 * registers of the device
860 rc = hl_hw_queues_create(hdev);
862 dev_err(hdev->dev, "failed to initialize kernel queues\n");
867 * Initialize the completion queues. Must be done before hw_init,
868 * because there the addresses of the completion queues are being
869 * passed as arguments to request_irq
871 hdev->completion_queue =
872 kcalloc(hdev->asic_prop.completion_queues_count,
873 sizeof(*hdev->completion_queue), GFP_KERNEL);
875 if (!hdev->completion_queue) {
876 dev_err(hdev->dev, "failed to allocate completion queues\n");
878 goto hw_queues_destroy;
881 for (i = 0, cq_ready_cnt = 0;
882 i < hdev->asic_prop.completion_queues_count;
883 i++, cq_ready_cnt++) {
884 rc = hl_cq_init(hdev, &hdev->completion_queue[i], i);
887 "failed to initialize completion queue\n");
893 * Initialize the event queue. Must be done before hw_init,
894 * because there the address of the event queue is being
895 * passed as argument to request_irq
897 rc = hl_eq_init(hdev, &hdev->event_queue);
899 dev_err(hdev->dev, "failed to initialize event queue\n");
903 /* Allocate the kernel context */
904 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
905 if (!hdev->kernel_ctx) {
910 hdev->user_ctx = NULL;
912 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
914 dev_err(hdev->dev, "failed to initialize kernel context\n");
918 rc = hl_cb_pool_init(hdev);
920 dev_err(hdev->dev, "failed to initialize CB pool\n");
924 rc = hl_sysfs_init(hdev);
926 dev_err(hdev->dev, "failed to initialize sysfs\n");
930 hl_debugfs_add_device(hdev);
932 if (hdev->asic_funcs->get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
934 "H/W state is dirty, must reset before initializing\n");
935 hdev->asic_funcs->hw_fini(hdev, true);
938 rc = hdev->asic_funcs->hw_init(hdev);
940 dev_err(hdev->dev, "failed to initialize the H/W\n");
945 hdev->disabled = false;
947 /* Check that the communication with the device is working */
948 rc = hdev->asic_funcs->test_queues(hdev);
950 dev_err(hdev->dev, "Failed to detect if device is alive\n");
955 /* After test_queues, KMD can start sending messages to device CPU */
957 rc = device_late_init(hdev);
959 dev_err(hdev->dev, "Failed late initialization\n");
964 dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
966 hdev->asic_prop.dram_size / 1024 / 1024 / 1024);
968 rc = hl_vm_init(hdev);
970 dev_err(hdev->dev, "Failed to initialize memory module\n");
976 * hl_hwmon_init must be called after device_late_init, because only
977 * there we get the information from the device about which
978 * hwmon-related sensors the device supports
980 rc = hl_hwmon_init(hdev);
982 dev_err(hdev->dev, "Failed to initialize hwmon\n");
987 dev_notice(hdev->dev,
988 "Successfully added device to habanalabs driver\n");
990 hdev->init_done = true;
995 hl_cb_pool_fini(hdev);
997 if (hl_ctx_put(hdev->kernel_ctx) != 1)
999 "kernel ctx is still alive on initialization failure\n");
1001 kfree(hdev->kernel_ctx);
1003 hl_eq_fini(hdev, &hdev->event_queue);
1005 for (i = 0 ; i < cq_ready_cnt ; i++)
1006 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1007 kfree(hdev->completion_queue);
1009 hl_hw_queues_destroy(hdev);
1011 hdev->asic_funcs->sw_fini(hdev);
1013 device_early_fini(hdev);
1015 device_destroy(hclass, hdev->dev->devt);
1016 cdev_del(&hdev->cdev);
1018 hdev->disabled = true;
1020 dev_err(&hdev->pdev->dev,
1021 "Failed to initialize hl%d. Device is NOT usable !\n",
1024 pr_err("Failed to initialize hl%d. Device is NOT usable !\n",
1031 * hl_device_fini - main tear-down function for habanalabs device
1033 * @hdev: pointer to habanalabs device structure
1035 * Destroy the device, call ASIC fini functions and release the id
1037 void hl_device_fini(struct hl_device *hdev)
1042 dev_info(hdev->dev, "Removing device\n");
1045 * This function is competing with the reset function, so try to
1046 * take the reset atomic and if we are already in middle of reset,
1047 * wait until reset function is finished. Reset function is designed
1048 * to always finish (could take up to a few seconds in worst case).
1051 timeout = ktime_add_us(ktime_get(),
1052 HL_PENDING_RESET_PER_SEC * 1000 * 1000 * 4);
1053 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1055 usleep_range(50, 200);
1056 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1057 if (ktime_compare(ktime_get(), timeout) > 0) {
1058 WARN(1, "Failed to remove device because reset function did not finish\n");
1063 /* Mark device as disabled */
1064 hdev->disabled = true;
1067 * Flush anyone that is inside the critical section of enqueue
1070 hdev->asic_funcs->hw_queues_lock(hdev);
1071 hdev->asic_funcs->hw_queues_unlock(hdev);
1073 hdev->hard_reset_pending = true;
1075 device_kill_open_processes(hdev);
1077 hl_hwmon_fini(hdev);
1079 device_late_fini(hdev);
1081 hl_debugfs_remove_device(hdev);
1083 hl_sysfs_fini(hdev);
1086 * Halt the engines and disable interrupts so we won't get any more
1087 * completions from H/W and we won't have any accesses from the
1088 * H/W to the host machine
1090 hdev->asic_funcs->halt_engines(hdev, true);
1092 /* Go over all the queues, release all CS and their jobs */
1093 hl_cs_rollback_all(hdev);
1095 hl_cb_pool_fini(hdev);
1097 /* Release kernel context */
1098 if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
1099 dev_err(hdev->dev, "kernel ctx is still alive\n");
1101 /* Reset the H/W. It will be in idle state after this returns */
1102 hdev->asic_funcs->hw_fini(hdev, true);
1106 hl_eq_fini(hdev, &hdev->event_queue);
1108 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1109 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1110 kfree(hdev->completion_queue);
1112 hl_hw_queues_destroy(hdev);
1114 /* Call ASIC S/W finalize function */
1115 hdev->asic_funcs->sw_fini(hdev);
1117 device_early_fini(hdev);
1119 /* Hide device from user */
1120 device_destroy(hdev->dev->class, hdev->dev->devt);
1121 cdev_del(&hdev->cdev);
1123 pr_info("removed device successfully\n");
1127 * hl_poll_timeout_memory - Periodically poll a host memory address
1128 * until it is not zero or a timeout occurs
1129 * @hdev: pointer to habanalabs device structure
1130 * @addr: Address to poll
1131 * @timeout_us: timeout in us
1132 * @val: Variable to read the value into
1134 * Returns 0 on success and -ETIMEDOUT upon a timeout. In either
1135 * case, the last read value at @addr is stored in @val. Must not
1136 * be called from atomic context if sleep_us or timeout_us are used.
1138 * The function sleeps for 100us with timeout value of
1141 int hl_poll_timeout_memory(struct hl_device *hdev, u64 addr,
1142 u32 timeout_us, u32 *val)
1145 * address in this function points always to a memory location in the
1146 * host's (server's) memory. That location is updated asynchronously
1147 * either by the direct access of the device or by another core
1149 u32 *paddr = (u32 *) (uintptr_t) addr;
1152 /* timeout should be longer when working with simulator */
1156 timeout = ktime_add_us(ktime_get(), timeout_us);
1162 * Flush CPU read/write buffers to make sure we read updates
1163 * done by other cores or by the device
1169 if (ktime_compare(ktime_get(), timeout) > 0) {
1173 usleep_range((100 >> 2) + 1, 100);
1176 return *val ? 0 : -ETIMEDOUT;
1180 * hl_poll_timeout_devicememory - Periodically poll a device memory address
1181 * until it is not zero or a timeout occurs
1182 * @hdev: pointer to habanalabs device structure
1183 * @addr: Device address to poll
1184 * @timeout_us: timeout in us
1185 * @val: Variable to read the value into
1187 * Returns 0 on success and -ETIMEDOUT upon a timeout. In either
1188 * case, the last read value at @addr is stored in @val. Must not
1189 * be called from atomic context if sleep_us or timeout_us are used.
1191 * The function sleeps for 100us with timeout value of
1194 int hl_poll_timeout_device_memory(struct hl_device *hdev, void __iomem *addr,
1195 u32 timeout_us, u32 *val)
1197 ktime_t timeout = ktime_add_us(ktime_get(), timeout_us);
1205 if (ktime_compare(ktime_get(), timeout) > 0) {
1209 usleep_range((100 >> 2) + 1, 100);
1212 return *val ? 0 : -ETIMEDOUT;
1216 * MMIO register access helper functions.
1220 * hl_rreg - Read an MMIO register
1222 * @hdev: pointer to habanalabs device structure
1223 * @reg: MMIO register offset (in bytes)
1225 * Returns the value of the MMIO register we are asked to read
1228 inline u32 hl_rreg(struct hl_device *hdev, u32 reg)
1230 return readl(hdev->rmmio + reg);
1234 * hl_wreg - Write to an MMIO register
1236 * @hdev: pointer to habanalabs device structure
1237 * @reg: MMIO register offset (in bytes)
1238 * @val: 32-bit value
1240 * Writes the 32-bit value into the MMIO register
1243 inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)
1245 writel(val, hdev->rmmio + reg);