habanalabs: improve IOCTLs behavior when disabled or reset

This patch makes some improvement in how IOCTLs behave when the device is
disabled or under reset.

The new code checks, at the start of every IOCTL, if the device is
disabled or in reset. If so, it prints an appropriate kernel message and
returns -EBUSY to user-space.

In addition, the code modifies the location of where the
hard_reset_pending flag is being set or cleared:

1. It is now cleared immediately after the reset *tear-down* flow is
   finished but before the re-initialization flow begins.

2. It is being set in the remove function of the device, to make the
   behavior the same with the hard-reset flow

There are two exceptions to the disable or in reset check:

1. The HL_INFO_DEVICE_STATUS opcode in the INFO IOCTL. This opcode allows
   the user to inquire about the status of the device, whether it is
   operational, in reset or malfunction (disabled). If the driver will
   block this IOCTL, the user won't be able to retrieve the status in
   case of malfunction or in reset.

2. The WAIT_FOR_CS IOCTL. This IOCTL allows the user to inquire about the
   status of a CS. We want to allow the user to continue to do so, even if
   we started a soft-reset process because it will allow the user to get
   the correct error code for each CS he submitted.

Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>

diff --git a/drivers/misc/habanalabs/command_buffer.c b/drivers/misc/habanalabs/command_buffer.c
index 85f75806a9a7ddb1ab2a26ccec4d8a3b7f04678a..b1ffca47d7484b81377e4e519149b6c1ca019599 100644 (file)
--- a/drivers/misc/habanalabs/command_buffer.c
+++ b/drivers/misc/habanalabs/command_buffer.c
@@ -214,6 +214,13 @@ int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
        u64 handle;
        int rc;
 
+       if (hl_device_disabled_or_in_reset(hdev)) {
+               dev_warn_ratelimited(hdev->dev,
+                       "Device is %s. Can't execute CB IOCTL\n",
+                       atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
+               return -EBUSY;
+       }
+
        switch (args->in.op) {
        case HL_CB_OP_CREATE:
                rc = hl_cb_create(hdev, &hpriv->cb_mgr, args->in.cb_size,

diff --git a/drivers/misc/habanalabs/device.c b/drivers/misc/habanalabs/device.c
index 56d3ba53c661810e566bd6961772b9156d87f68a..25bfb093ff26fe47dccb277725fc2a5af13e9b70 100644 (file)
--- a/drivers/misc/habanalabs/device.c
+++ b/drivers/misc/habanalabs/device.c
@@ -641,6 +641,8 @@ int hl_device_reset(struct hl_device *hdev, bool hard_reset,
        if ((hard_reset) && (!from_hard_reset_thread)) {
                struct hl_device_reset_work *device_reset_work;
 
+               hdev->hard_reset_pending = true;
+
                if (!hdev->pdev) {
                        dev_err(hdev->dev,
                                "Reset action is NOT supported in simulator\n");
@@ -648,8 +650,6 @@ int hl_device_reset(struct hl_device *hdev, bool hard_reset,
                        goto out_err;
                }
 
-               hdev->hard_reset_pending = true;
-
                device_reset_work = kzalloc(sizeof(*device_reset_work),
                                                GFP_ATOMIC);
                if (!device_reset_work) {
@@ -718,6 +718,7 @@ int hl_device_reset(struct hl_device *hdev, bool hard_reset,
 
        if (hard_reset) {
                hdev->device_cpu_disabled = false;
+               hdev->hard_reset_pending = false;
 
                if (hdev->kernel_ctx) {
                        dev_crit(hdev->dev,
@@ -779,8 +780,6 @@ int hl_device_reset(struct hl_device *hdev, bool hard_reset,
                }
 
                hl_set_max_power(hdev, hdev->max_power);
-
-               hdev->hard_reset_pending = false;
        } else {
                rc = hdev->asic_funcs->soft_reset_late_init(hdev);
                if (rc) {
@@ -1069,6 +1068,8 @@ void hl_device_fini(struct hl_device *hdev)
        hdev->asic_funcs->hw_queues_lock(hdev);
        hdev->asic_funcs->hw_queues_unlock(hdev);
 
+       hdev->hard_reset_pending = true;
+
        device_kill_open_processes(hdev);
 
        hl_hwmon_fini(hdev);

diff --git a/drivers/misc/habanalabs/habanalabs_ioctl.c b/drivers/misc/habanalabs/habanalabs_ioctl.c
index 810e6544607535be1332cdb33d5f40ffec43f0d2..eeefb22023e9a1cac01c46dc196cccf98574629a 100644 (file)
--- a/drivers/misc/habanalabs/habanalabs_ioctl.c
+++ b/drivers/misc/habanalabs/habanalabs_ioctl.c
@@ -202,7 +202,8 @@ static int hl_info_ioctl(struct hl_fpriv *hpriv, void *data)
 
        if (hl_device_disabled_or_in_reset(hdev)) {
                dev_warn_ratelimited(hdev->dev,
-                       "Device is disabled or in reset. Can't execute INFO IOCTL\n");
+                       "Device is %s. Can't execute INFO IOCTL\n",
+                       atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
                return -EBUSY;
        }
 

diff --git a/drivers/misc/habanalabs/memory.c b/drivers/misc/habanalabs/memory.c
index 05919f9133008752e52ad96e52534aac05c46610..43ef3ad8438ab6dc2d276237e95df2b17e44ec99 100644 (file)
--- a/drivers/misc/habanalabs/memory.c
+++ b/drivers/misc/habanalabs/memory.c
@@ -1159,7 +1159,8 @@ int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data)
 
        if (hl_device_disabled_or_in_reset(hdev)) {
                dev_warn_ratelimited(hdev->dev,
-                       "Device is disabled or in reset. Can't execute memory IOCTL\n");
+                       "Device is %s. Can't execute MEMORY IOCTL\n",
+                       atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
                return -EBUSY;
        }