1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
9 #define pr_fmt(fmt) "habanalabs: " fmt
11 #include "habanalabs.h"
13 #include <linux/pci.h>
14 #include <linux/module.h>
16 #define HL_DRIVER_AUTHOR "HabanaLabs Kernel Driver Team"
18 #define HL_DRIVER_DESC "Driver for HabanaLabs's AI Accelerators"
20 MODULE_AUTHOR(HL_DRIVER_AUTHOR);
21 MODULE_DESCRIPTION(HL_DRIVER_DESC);
22 MODULE_LICENSE("GPL v2");
25 static struct class *hl_class;
26 static DEFINE_IDR(hl_devs_idr);
27 static DEFINE_MUTEX(hl_devs_idr_lock);
29 static int timeout_locked = 5;
30 static int reset_on_lockup = 1;
32 module_param(timeout_locked, int, 0444);
33 MODULE_PARM_DESC(timeout_locked,
34 "Device lockup timeout in seconds (0 = disabled, default 5s)");
36 module_param(reset_on_lockup, int, 0444);
37 MODULE_PARM_DESC(reset_on_lockup,
38 "Do device reset on lockup (0 = no, 1 = yes, default yes)");
40 #define PCI_VENDOR_ID_HABANALABS 0x1da3
42 #define PCI_IDS_GOYA 0x0001
44 static const struct pci_device_id ids[] = {
45 { PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GOYA), },
48 MODULE_DEVICE_TABLE(pci, ids);
51 * get_asic_type - translate device id to asic type
53 * @device: id of the PCI device
55 * Translate device id to asic type.
56 * In case of unidentified device, return -1
58 static enum hl_asic_type get_asic_type(u16 device)
60 enum hl_asic_type asic_type;
64 asic_type = ASIC_GOYA;
67 asic_type = ASIC_INVALID;
75 * hl_device_open - open function for habanalabs device
77 * @inode: pointer to inode structure
78 * @filp: pointer to file structure
80 * Called when process opens an habanalabs device.
82 int hl_device_open(struct inode *inode, struct file *filp)
84 struct hl_device *hdev;
85 struct hl_fpriv *hpriv;
88 mutex_lock(&hl_devs_idr_lock);
89 hdev = idr_find(&hl_devs_idr, iminor(inode));
90 mutex_unlock(&hl_devs_idr_lock);
93 pr_err("Couldn't find device %d:%d\n",
94 imajor(inode), iminor(inode));
98 mutex_lock(&hdev->fd_open_cnt_lock);
100 if (hl_device_disabled_or_in_reset(hdev)) {
101 dev_err_ratelimited(hdev->dev,
102 "Can't open %s because it is disabled or in reset\n",
103 dev_name(hdev->dev));
104 mutex_unlock(&hdev->fd_open_cnt_lock);
108 if (atomic_read(&hdev->fd_open_cnt)) {
109 dev_info_ratelimited(hdev->dev,
110 "Device %s is already attached to application\n",
111 dev_name(hdev->dev));
112 mutex_unlock(&hdev->fd_open_cnt_lock);
116 atomic_inc(&hdev->fd_open_cnt);
118 mutex_unlock(&hdev->fd_open_cnt_lock);
120 hpriv = kzalloc(sizeof(*hpriv), GFP_KERNEL);
127 filp->private_data = hpriv;
129 mutex_init(&hpriv->restore_phase_mutex);
130 kref_init(&hpriv->refcount);
131 nonseekable_open(inode, filp);
133 hl_cb_mgr_init(&hpriv->cb_mgr);
134 hl_ctx_mgr_init(&hpriv->ctx_mgr);
136 rc = hl_ctx_create(hdev, hpriv);
138 dev_err(hdev->dev, "Failed to open FD (CTX fail)\n");
142 hpriv->taskpid = find_get_pid(current->pid);
145 * Device is IDLE at this point so it is legal to change PLLs. There
146 * is no need to check anything because if the PLL is already HIGH, the
147 * set function will return without doing anything
149 hl_device_set_frequency(hdev, PLL_HIGH);
151 hl_debugfs_add_file(hpriv);
156 filp->private_data = NULL;
157 hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
158 hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);
159 mutex_destroy(&hpriv->restore_phase_mutex);
163 atomic_dec(&hdev->fd_open_cnt);
168 * create_hdev - create habanalabs device instance
170 * @dev: will hold the pointer to the new habanalabs device structure
171 * @pdev: pointer to the pci device
172 * @asic_type: in case of simulator device, which device is it
173 * @minor: in case of simulator device, the minor of the device
175 * Allocate memory for habanalabs device and initialize basic fields
176 * Identify the ASIC type
177 * Allocate ID (minor) for the device (only for real devices)
179 int create_hdev(struct hl_device **dev, struct pci_dev *pdev,
180 enum hl_asic_type asic_type, int minor)
182 struct hl_device *hdev;
187 hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
191 hdev->major = hl_major;
192 hdev->reset_on_lockup = reset_on_lockup;
194 /* Parameters for bring-up - set them to defaults */
195 hdev->mmu_enable = 1;
196 hdev->cpu_enable = 1;
197 hdev->reset_pcilink = 0;
198 hdev->cpu_queues_enable = 1;
199 hdev->fw_loading = 1;
203 /* If CPU is disabled, no point in loading FW */
204 if (!hdev->cpu_enable)
205 hdev->fw_loading = 0;
207 /* If we don't load FW, no need to initialize CPU queues */
208 if (!hdev->fw_loading)
209 hdev->cpu_queues_enable = 0;
211 /* If CPU queues not enabled, no way to do heartbeat */
212 if (!hdev->cpu_queues_enable)
216 hdev->timeout_jiffies = msecs_to_jiffies(timeout_locked * 1000);
218 hdev->timeout_jiffies = MAX_SCHEDULE_TIMEOUT;
220 hdev->disabled = true;
221 hdev->pdev = pdev; /* can be NULL in case of simulator device */
224 hdev->asic_type = get_asic_type(pdev->device);
225 if (hdev->asic_type == ASIC_INVALID) {
226 dev_err(&pdev->dev, "Unsupported ASIC\n");
231 hdev->asic_type = asic_type;
234 /* Set default DMA mask to 32 bits */
237 mutex_lock(&hl_devs_idr_lock);
240 rc = idr_alloc(&hl_devs_idr, hdev, 0, HL_MAX_MINORS,
243 void *old_idr = idr_replace(&hl_devs_idr, hdev, minor);
245 if (IS_ERR_VALUE(old_idr)) {
246 rc = PTR_ERR(old_idr);
247 pr_err("Error %d when trying to replace minor %d\n",
249 mutex_unlock(&hl_devs_idr_lock);
255 mutex_unlock(&hl_devs_idr_lock);
259 pr_err("too many devices in the system\n");
277 * destroy_hdev - destroy habanalabs device instance
279 * @dev: pointer to the habanalabs device structure
282 void destroy_hdev(struct hl_device *hdev)
284 /* Remove device from the device list */
285 mutex_lock(&hl_devs_idr_lock);
286 idr_remove(&hl_devs_idr, hdev->id);
287 mutex_unlock(&hl_devs_idr_lock);
292 static int hl_pmops_suspend(struct device *dev)
294 struct pci_dev *pdev = to_pci_dev(dev);
295 struct hl_device *hdev = pci_get_drvdata(pdev);
297 pr_debug("Going to suspend PCI device\n");
300 pr_err("device pointer is NULL in suspend\n");
304 return hl_device_suspend(hdev);
307 static int hl_pmops_resume(struct device *dev)
309 struct pci_dev *pdev = to_pci_dev(dev);
310 struct hl_device *hdev = pci_get_drvdata(pdev);
312 pr_debug("Going to resume PCI device\n");
315 pr_err("device pointer is NULL in resume\n");
319 return hl_device_resume(hdev);
323 * hl_pci_probe - probe PCI habanalabs devices
325 * @pdev: pointer to pci device
326 * @id: pointer to pci device id structure
328 * Standard PCI probe function for habanalabs device.
329 * Create a new habanalabs device and initialize it according to the
332 static int hl_pci_probe(struct pci_dev *pdev,
333 const struct pci_device_id *id)
335 struct hl_device *hdev;
338 dev_info(&pdev->dev, HL_NAME
339 " device found [%04x:%04x] (rev %x)\n",
340 (int)pdev->vendor, (int)pdev->device, (int)pdev->revision);
342 rc = create_hdev(&hdev, pdev, ASIC_INVALID, -1);
346 pci_set_drvdata(pdev, hdev);
348 rc = hl_device_init(hdev, hl_class);
350 dev_err(&pdev->dev, "Fatal error during habanalabs device init\n");
358 pci_set_drvdata(pdev, NULL);
365 * hl_pci_remove - remove PCI habanalabs devices
367 * @pdev: pointer to pci device
369 * Standard PCI remove function for habanalabs device
371 static void hl_pci_remove(struct pci_dev *pdev)
373 struct hl_device *hdev;
375 hdev = pci_get_drvdata(pdev);
379 hl_device_fini(hdev);
380 pci_set_drvdata(pdev, NULL);
385 static const struct dev_pm_ops hl_pm_ops = {
386 .suspend = hl_pmops_suspend,
387 .resume = hl_pmops_resume,
390 static struct pci_driver hl_pci_driver = {
393 .probe = hl_pci_probe,
394 .remove = hl_pci_remove,
395 .driver.pm = &hl_pm_ops,
399 * hl_init - Initialize the habanalabs kernel driver
401 static int __init hl_init(void)
406 pr_info("loading driver\n");
408 rc = alloc_chrdev_region(&dev, 0, HL_MAX_MINORS, HL_NAME);
410 pr_err("unable to get major\n");
414 hl_major = MAJOR(dev);
416 hl_class = class_create(THIS_MODULE, HL_NAME);
417 if (IS_ERR(hl_class)) {
418 pr_err("failed to allocate class\n");
419 rc = PTR_ERR(hl_class);
425 rc = pci_register_driver(&hl_pci_driver);
427 pr_err("failed to register pci device\n");
431 pr_debug("driver loaded\n");
437 class_destroy(hl_class);
439 unregister_chrdev_region(MKDEV(hl_major, 0), HL_MAX_MINORS);
444 * hl_exit - Release all resources of the habanalabs kernel driver
446 static void __exit hl_exit(void)
448 pci_unregister_driver(&hl_pci_driver);
451 * Removing debugfs must be after all devices or simulator devices
452 * have been removed because otherwise we get a bug in the
453 * debugfs module for referencing NULL objects
457 class_destroy(hl_class);
458 unregister_chrdev_region(MKDEV(hl_major, 0), HL_MAX_MINORS);
460 idr_destroy(&hl_devs_idr);
462 pr_debug("driver removed\n");
465 module_init(hl_init);
466 module_exit(hl_exit);