drivers/pci/switch/switchtec.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Microsemi Switchtec(tm) PCIe Management Driver
   4  * Copyright (c) 2017, Microsemi Corporation
   5  */
   6
   7 #include <linux/switchtec.h>
   8 #include <linux/switchtec_ioctl.h>
   9
  10 #include <linux/interrupt.h>
  11 #include <linux/module.h>
  12 #include <linux/fs.h>
  13 #include <linux/uaccess.h>
  14 #include <linux/poll.h>
  15 #include <linux/wait.h>
  16 #include <linux/io-64-nonatomic-lo-hi.h>
  17 #include <linux/nospec.h>
  18
  19 MODULE_DESCRIPTION("Microsemi Switchtec(tm) PCIe Management Driver");
  20 MODULE_VERSION("0.1");
  21 MODULE_LICENSE("GPL");
  22 MODULE_AUTHOR("Microsemi Corporation");
  23
  24 static int max_devices = 16;
  25 module_param(max_devices, int, 0644);
  26 MODULE_PARM_DESC(max_devices, "max number of switchtec device instances");
  27
  28 static bool use_dma_mrpc = 1;
  29 module_param(use_dma_mrpc, bool, 0644);
  30 MODULE_PARM_DESC(use_dma_mrpc,
  31                  "Enable the use of the DMA MRPC feature");
  32
  33 static int nirqs = 32;
  34 module_param(nirqs, int, 0644);
  35 MODULE_PARM_DESC(nirqs, "number of interrupts to allocate (more may be useful for NTB applications)");
  36
  37 static dev_t switchtec_devt;
  38 static DEFINE_IDA(switchtec_minor_ida);
  39
  40 struct class *switchtec_class;
  41 EXPORT_SYMBOL_GPL(switchtec_class);
  42
  43 enum mrpc_state {
  44         MRPC_IDLE = 0,
  45         MRPC_QUEUED,
  46         MRPC_RUNNING,
  47         MRPC_DONE,
  48 };
  49
  50 struct switchtec_user {
  51         struct switchtec_dev *stdev;
  52
  53         enum mrpc_state state;
  54
  55         struct completion comp;
  56         struct kref kref;
  57         struct list_head list;
  58
  59         u32 cmd;
  60         u32 status;
  61         u32 return_code;
  62         size_t data_len;
  63         size_t read_len;
  64         unsigned char data[SWITCHTEC_MRPC_PAYLOAD_SIZE];
  65         int event_cnt;
  66 };
  67
  68 static struct switchtec_user *stuser_create(struct switchtec_dev *stdev)
  69 {
  70         struct switchtec_user *stuser;
  71
  72         stuser = kzalloc(sizeof(*stuser), GFP_KERNEL);
  73         if (!stuser)
  74                 return ERR_PTR(-ENOMEM);
  75
  76         get_device(&stdev->dev);
  77         stuser->stdev = stdev;
  78         kref_init(&stuser->kref);
  79         INIT_LIST_HEAD(&stuser->list);
  80         init_completion(&stuser->comp);
  81         stuser->event_cnt = atomic_read(&stdev->event_cnt);
  82
  83         dev_dbg(&stdev->dev, "%s: %p\n", __func__, stuser);
  84
  85         return stuser;
  86 }
  87
  88 static void stuser_free(struct kref *kref)
  89 {
  90         struct switchtec_user *stuser;
  91
  92         stuser = container_of(kref, struct switchtec_user, kref);
  93
  94         dev_dbg(&stuser->stdev->dev, "%s: %p\n", __func__, stuser);
  95
  96         put_device(&stuser->stdev->dev);
  97         kfree(stuser);
  98 }
  99
 100 static void stuser_put(struct switchtec_user *stuser)
 101 {
 102         kref_put(&stuser->kref, stuser_free);
 103 }
 104
 105 static void stuser_set_state(struct switchtec_user *stuser,
 106                              enum mrpc_state state)
 107 {
 108         /* requires the mrpc_mutex to already be held when called */
 109
 110         const char * const state_names[] = {
 111                 [MRPC_IDLE] = "IDLE",
 112                 [MRPC_QUEUED] = "QUEUED",
 113                 [MRPC_RUNNING] = "RUNNING",
 114                 [MRPC_DONE] = "DONE",
 115         };
 116
 117         stuser->state = state;
 118
 119         dev_dbg(&stuser->stdev->dev, "stuser state %p -> %s",
 120                 stuser, state_names[state]);
 121 }
 122
 123 static void mrpc_complete_cmd(struct switchtec_dev *stdev);
 124
 125 static void flush_wc_buf(struct switchtec_dev *stdev)
 126 {
 127         struct ntb_dbmsg_regs __iomem *mmio_dbmsg;
 128
 129         /*
 130          * odb (outbound doorbell) register is processed by low latency
 131          * hardware and w/o side effect
 132          */
 133         mmio_dbmsg = (void __iomem *)stdev->mmio_ntb +
 134                 SWITCHTEC_NTB_REG_DBMSG_OFFSET;
 135         ioread32(&mmio_dbmsg->odb);
 136 }
 137
 138 static void mrpc_cmd_submit(struct switchtec_dev *stdev)
 139 {
 140         /* requires the mrpc_mutex to already be held when called */
 141
 142         struct switchtec_user *stuser;
 143
 144         if (stdev->mrpc_busy)
 145                 return;
 146
 147         if (list_empty(&stdev->mrpc_queue))
 148                 return;
 149
 150         stuser = list_entry(stdev->mrpc_queue.next, struct switchtec_user,
 151                             list);
 152
 153         if (stdev->dma_mrpc) {
 154                 stdev->dma_mrpc->status = SWITCHTEC_MRPC_STATUS_INPROGRESS;
 155                 memset(stdev->dma_mrpc->data, 0xFF, SWITCHTEC_MRPC_PAYLOAD_SIZE);
 156         }
 157
 158         stuser_set_state(stuser, MRPC_RUNNING);
 159         stdev->mrpc_busy = 1;
 160         memcpy_toio(&stdev->mmio_mrpc->input_data,
 161                     stuser->data, stuser->data_len);
 162         flush_wc_buf(stdev);
 163         iowrite32(stuser->cmd, &stdev->mmio_mrpc->cmd);
 164
 165         schedule_delayed_work(&stdev->mrpc_timeout,
 166                               msecs_to_jiffies(500));
 167 }
 168
 169 static int mrpc_queue_cmd(struct switchtec_user *stuser)
 170 {
 171         /* requires the mrpc_mutex to already be held when called */
 172
 173         struct switchtec_dev *stdev = stuser->stdev;
 174
 175         kref_get(&stuser->kref);
 176         stuser->read_len = sizeof(stuser->data);
 177         stuser_set_state(stuser, MRPC_QUEUED);
 178         init_completion(&stuser->comp);
 179         list_add_tail(&stuser->list, &stdev->mrpc_queue);
 180
 181         mrpc_cmd_submit(stdev);
 182
 183         return 0;
 184 }
 185
 186 static void mrpc_complete_cmd(struct switchtec_dev *stdev)
 187 {
 188         /* requires the mrpc_mutex to already be held when called */
 189         struct switchtec_user *stuser;
 190
 191         if (list_empty(&stdev->mrpc_queue))
 192                 return;
 193
 194         stuser = list_entry(stdev->mrpc_queue.next, struct switchtec_user,
 195                             list);
 196
 197         if (stdev->dma_mrpc)
 198                 stuser->status = stdev->dma_mrpc->status;
 199         else
 200                 stuser->status = ioread32(&stdev->mmio_mrpc->status);
 201
 202         if (stuser->status == SWITCHTEC_MRPC_STATUS_INPROGRESS)
 203                 return;
 204
 205         stuser_set_state(stuser, MRPC_DONE);
 206         stuser->return_code = 0;
 207
 208         if (stuser->status != SWITCHTEC_MRPC_STATUS_DONE)
 209                 goto out;
 210
 211         if (stdev->dma_mrpc)
 212                 stuser->return_code = stdev->dma_mrpc->rtn_code;
 213         else
 214                 stuser->return_code = ioread32(&stdev->mmio_mrpc->ret_value);
 215         if (stuser->return_code != 0)
 216                 goto out;
 217
 218         if (stdev->dma_mrpc)
 219                 memcpy(stuser->data, &stdev->dma_mrpc->data,
 220                               stuser->read_len);
 221         else
 222                 memcpy_fromio(stuser->data, &stdev->mmio_mrpc->output_data,
 223                               stuser->read_len);
 224 out:
 225         complete_all(&stuser->comp);
 226         list_del_init(&stuser->list);
 227         stuser_put(stuser);
 228         stdev->mrpc_busy = 0;
 229
 230         mrpc_cmd_submit(stdev);
 231 }
 232
 233 static void mrpc_event_work(struct work_struct *work)
 234 {
 235         struct switchtec_dev *stdev;
 236
 237         stdev = container_of(work, struct switchtec_dev, mrpc_work);
 238
 239         dev_dbg(&stdev->dev, "%s\n", __func__);
 240
 241         mutex_lock(&stdev->mrpc_mutex);
 242         cancel_delayed_work(&stdev->mrpc_timeout);
 243         mrpc_complete_cmd(stdev);
 244         mutex_unlock(&stdev->mrpc_mutex);
 245 }
 246
 247 static void mrpc_timeout_work(struct work_struct *work)
 248 {
 249         struct switchtec_dev *stdev;
 250         u32 status;
 251
 252         stdev = container_of(work, struct switchtec_dev, mrpc_timeout.work);
 253
 254         dev_dbg(&stdev->dev, "%s\n", __func__);
 255
 256         mutex_lock(&stdev->mrpc_mutex);
 257
 258         if (stdev->dma_mrpc)
 259                 status = stdev->dma_mrpc->status;
 260         else
 261                 status = ioread32(&stdev->mmio_mrpc->status);
 262         if (status == SWITCHTEC_MRPC_STATUS_INPROGRESS) {
 263                 schedule_delayed_work(&stdev->mrpc_timeout,
 264                                       msecs_to_jiffies(500));
 265                 goto out;
 266         }
 267
 268         mrpc_complete_cmd(stdev);
 269 out:
 270         mutex_unlock(&stdev->mrpc_mutex);
 271 }
 272
 273 static ssize_t device_version_show(struct device *dev,
 274         struct device_attribute *attr, char *buf)
 275 {
 276         struct switchtec_dev *stdev = to_stdev(dev);
 277         u32 ver;
 278
 279         ver = ioread32(&stdev->mmio_sys_info->device_version);
 280
 281         return sprintf(buf, "%x\n", ver);
 282 }
 283 static DEVICE_ATTR_RO(device_version);
 284
 285 static ssize_t fw_version_show(struct device *dev,
 286         struct device_attribute *attr, char *buf)
 287 {
 288         struct switchtec_dev *stdev = to_stdev(dev);
 289         u32 ver;
 290
 291         ver = ioread32(&stdev->mmio_sys_info->firmware_version);
 292
 293         return sprintf(buf, "%08x\n", ver);
 294 }
 295 static DEVICE_ATTR_RO(fw_version);
 296
 297 static ssize_t io_string_show(char *buf, void __iomem *attr, size_t len)
 298 {
 299         int i;
 300
 301         memcpy_fromio(buf, attr, len);
 302         buf[len] = '\n';
 303         buf[len + 1] = 0;
 304
 305         for (i = len - 1; i > 0; i--) {
 306                 if (buf[i] != ' ')
 307                         break;
 308                 buf[i] = '\n';
 309                 buf[i + 1] = 0;
 310         }
 311
 312         return strlen(buf);
 313 }
 314
 315 #define DEVICE_ATTR_SYS_INFO_STR(field) \
 316 static ssize_t field ## _show(struct device *dev, \
 317         struct device_attribute *attr, char *buf) \
 318 { \
 319         struct switchtec_dev *stdev = to_stdev(dev); \
 320         struct sys_info_regs __iomem *si = stdev->mmio_sys_info; \
 321         if (stdev->gen == SWITCHTEC_GEN3) \
 322                 return io_string_show(buf, &si->gen3.field, \
 323                                       sizeof(si->gen3.field)); \
 324         else if (stdev->gen == SWITCHTEC_GEN4) \
 325                 return io_string_show(buf, &si->gen4.field, \
 326                                       sizeof(si->gen4.field)); \
 327         else \
 328                 return -ENOTSUPP; \
 329 } \
 330 \
 331 static DEVICE_ATTR_RO(field)
 332
 333 DEVICE_ATTR_SYS_INFO_STR(vendor_id);
 334 DEVICE_ATTR_SYS_INFO_STR(product_id);
 335 DEVICE_ATTR_SYS_INFO_STR(product_revision);
 336
 337 static ssize_t component_vendor_show(struct device *dev,
 338                                      struct device_attribute *attr, char *buf)
 339 {
 340         struct switchtec_dev *stdev = to_stdev(dev);
 341         struct sys_info_regs __iomem *si = stdev->mmio_sys_info;
 342
 343         /* component_vendor field not supported after gen3 */
 344         if (stdev->gen != SWITCHTEC_GEN3)
 345                 return sprintf(buf, "none\n");
 346
 347         return io_string_show(buf, &si->gen3.component_vendor,
 348                               sizeof(si->gen3.component_vendor));
 349 }
 350 static DEVICE_ATTR_RO(component_vendor);
 351
 352 static ssize_t component_id_show(struct device *dev,
 353         struct device_attribute *attr, char *buf)
 354 {
 355         struct switchtec_dev *stdev = to_stdev(dev);
 356         int id = ioread16(&stdev->mmio_sys_info->gen3.component_id);
 357
 358         /* component_id field not supported after gen3 */
 359         if (stdev->gen != SWITCHTEC_GEN3)
 360                 return sprintf(buf, "none\n");
 361
 362         return sprintf(buf, "PM%04X\n", id);
 363 }
 364 static DEVICE_ATTR_RO(component_id);
 365
 366 static ssize_t component_revision_show(struct device *dev,
 367         struct device_attribute *attr, char *buf)
 368 {
 369         struct switchtec_dev *stdev = to_stdev(dev);
 370         int rev = ioread8(&stdev->mmio_sys_info->gen3.component_revision);
 371
 372         /* component_revision field not supported after gen3 */
 373         if (stdev->gen != SWITCHTEC_GEN3)
 374                 return sprintf(buf, "255\n");
 375
 376         return sprintf(buf, "%d\n", rev);
 377 }
 378 static DEVICE_ATTR_RO(component_revision);
 379
 380 static ssize_t partition_show(struct device *dev,
 381         struct device_attribute *attr, char *buf)
 382 {
 383         struct switchtec_dev *stdev = to_stdev(dev);
 384
 385         return sprintf(buf, "%d\n", stdev->partition);
 386 }
 387 static DEVICE_ATTR_RO(partition);
 388
 389 static ssize_t partition_count_show(struct device *dev,
 390         struct device_attribute *attr, char *buf)
 391 {
 392         struct switchtec_dev *stdev = to_stdev(dev);
 393
 394         return sprintf(buf, "%d\n", stdev->partition_count);
 395 }
 396 static DEVICE_ATTR_RO(partition_count);
 397
 398 static struct attribute *switchtec_device_attrs[] = {
 399         &dev_attr_device_version.attr,
 400         &dev_attr_fw_version.attr,
 401         &dev_attr_vendor_id.attr,
 402         &dev_attr_product_id.attr,
 403         &dev_attr_product_revision.attr,
 404         &dev_attr_component_vendor.attr,
 405         &dev_attr_component_id.attr,
 406         &dev_attr_component_revision.attr,
 407         &dev_attr_partition.attr,
 408         &dev_attr_partition_count.attr,
 409         NULL,
 410 };
 411
 412 ATTRIBUTE_GROUPS(switchtec_device);
 413
 414 static int switchtec_dev_open(struct inode *inode, struct file *filp)
 415 {
 416         struct switchtec_dev *stdev;
 417         struct switchtec_user *stuser;
 418
 419         stdev = container_of(inode->i_cdev, struct switchtec_dev, cdev);
 420
 421         stuser = stuser_create(stdev);
 422         if (IS_ERR(stuser))
 423                 return PTR_ERR(stuser);
 424
 425         filp->private_data = stuser;
 426         stream_open(inode, filp);
 427
 428         dev_dbg(&stdev->dev, "%s: %p\n", __func__, stuser);
 429
 430         return 0;
 431 }
 432
 433 static int switchtec_dev_release(struct inode *inode, struct file *filp)
 434 {
 435         struct switchtec_user *stuser = filp->private_data;
 436
 437         stuser_put(stuser);
 438
 439         return 0;
 440 }
 441
 442 static int lock_mutex_and_test_alive(struct switchtec_dev *stdev)
 443 {
 444         if (mutex_lock_interruptible(&stdev->mrpc_mutex))
 445                 return -EINTR;
 446
 447         if (!stdev->alive) {
 448                 mutex_unlock(&stdev->mrpc_mutex);
 449                 return -ENODEV;
 450         }
 451
 452         return 0;
 453 }
 454
 455 static ssize_t switchtec_dev_write(struct file *filp, const char __user *data,
 456                                    size_t size, loff_t *off)
 457 {
 458         struct switchtec_user *stuser = filp->private_data;
 459         struct switchtec_dev *stdev = stuser->stdev;
 460         int rc;
 461
 462         if (size < sizeof(stuser->cmd) ||
 463             size > sizeof(stuser->cmd) + sizeof(stuser->data))
 464                 return -EINVAL;
 465
 466         stuser->data_len = size - sizeof(stuser->cmd);
 467
 468         rc = lock_mutex_and_test_alive(stdev);
 469         if (rc)
 470                 return rc;
 471
 472         if (stuser->state != MRPC_IDLE) {
 473                 rc = -EBADE;
 474                 goto out;
 475         }
 476
 477         rc = copy_from_user(&stuser->cmd, data, sizeof(stuser->cmd));
 478         if (rc) {
 479                 rc = -EFAULT;
 480                 goto out;
 481         }
 482
 483         data += sizeof(stuser->cmd);
 484         rc = copy_from_user(&stuser->data, data, size - sizeof(stuser->cmd));
 485         if (rc) {
 486                 rc = -EFAULT;
 487                 goto out;
 488         }
 489
 490         rc = mrpc_queue_cmd(stuser);
 491
 492 out:
 493         mutex_unlock(&stdev->mrpc_mutex);
 494
 495         if (rc)
 496                 return rc;
 497
 498         return size;
 499 }
 500
 501 static ssize_t switchtec_dev_read(struct file *filp, char __user *data,
 502                                   size_t size, loff_t *off)
 503 {
 504         struct switchtec_user *stuser = filp->private_data;
 505         struct switchtec_dev *stdev = stuser->stdev;
 506         int rc;
 507
 508         if (size < sizeof(stuser->cmd) ||
 509             size > sizeof(stuser->cmd) + sizeof(stuser->data))
 510                 return -EINVAL;
 511
 512         rc = lock_mutex_and_test_alive(stdev);
 513         if (rc)
 514                 return rc;
 515
 516         if (stuser->state == MRPC_IDLE) {
 517                 mutex_unlock(&stdev->mrpc_mutex);
 518                 return -EBADE;
 519         }
 520
 521         stuser->read_len = size - sizeof(stuser->return_code);
 522
 523         mutex_unlock(&stdev->mrpc_mutex);
 524
 525         if (filp->f_flags & O_NONBLOCK) {
 526                 if (!try_wait_for_completion(&stuser->comp))
 527                         return -EAGAIN;
 528         } else {
 529                 rc = wait_for_completion_interruptible(&stuser->comp);
 530                 if (rc < 0)
 531                         return rc;
 532         }
 533
 534         rc = lock_mutex_and_test_alive(stdev);
 535         if (rc)
 536                 return rc;
 537
 538         if (stuser->state != MRPC_DONE) {
 539                 mutex_unlock(&stdev->mrpc_mutex);
 540                 return -EBADE;
 541         }
 542
 543         rc = copy_to_user(data, &stuser->return_code,
 544                           sizeof(stuser->return_code));
 545         if (rc) {
 546                 rc = -EFAULT;
 547                 goto out;
 548         }
 549
 550         data += sizeof(stuser->return_code);
 551         rc = copy_to_user(data, &stuser->data,
 552                           size - sizeof(stuser->return_code));
 553         if (rc) {
 554                 rc = -EFAULT;
 555                 goto out;
 556         }
 557
 558         stuser_set_state(stuser, MRPC_IDLE);
 559
 560 out:
 561         mutex_unlock(&stdev->mrpc_mutex);
 562
 563         if (stuser->status == SWITCHTEC_MRPC_STATUS_DONE)
 564                 return size;
 565         else if (stuser->status == SWITCHTEC_MRPC_STATUS_INTERRUPTED)
 566                 return -ENXIO;
 567         else
 568                 return -EBADMSG;
 569 }
 570
 571 static __poll_t switchtec_dev_poll(struct file *filp, poll_table *wait)
 572 {
 573         struct switchtec_user *stuser = filp->private_data;
 574         struct switchtec_dev *stdev = stuser->stdev;
 575         __poll_t ret = 0;
 576
 577         poll_wait(filp, &stuser->comp.wait, wait);
 578         poll_wait(filp, &stdev->event_wq, wait);
 579
 580         if (lock_mutex_and_test_alive(stdev))
 581                 return EPOLLIN | EPOLLRDHUP | EPOLLOUT | EPOLLERR | EPOLLHUP;
 582
 583         mutex_unlock(&stdev->mrpc_mutex);
 584
 585         if (try_wait_for_completion(&stuser->comp))
 586                 ret |= EPOLLIN | EPOLLRDNORM;
 587
 588         if (stuser->event_cnt != atomic_read(&stdev->event_cnt))
 589                 ret |= EPOLLPRI | EPOLLRDBAND;
 590
 591         return ret;
 592 }
 593
 594 static int ioctl_flash_info(struct switchtec_dev *stdev,
 595                             struct switchtec_ioctl_flash_info __user *uinfo)
 596 {
 597         struct switchtec_ioctl_flash_info info = {0};
 598         struct flash_info_regs __iomem *fi = stdev->mmio_flash_info;
 599
 600         if (stdev->gen == SWITCHTEC_GEN3) {
 601                 info.flash_length = ioread32(&fi->gen3.flash_length);
 602                 info.num_partitions = SWITCHTEC_NUM_PARTITIONS_GEN3;
 603         } else {
 604                 return -ENOTSUPP;
 605         }
 606
 607         if (copy_to_user(uinfo, &info, sizeof(info)))
 608                 return -EFAULT;
 609
 610         return 0;
 611 }
 612
 613 static void set_fw_info_part(struct switchtec_ioctl_flash_part_info *info,
 614                              struct partition_info __iomem *pi)
 615 {
 616         info->address = ioread32(&pi->address);
 617         info->length = ioread32(&pi->length);
 618 }
 619
 620 static int flash_part_info_gen3(struct switchtec_dev *stdev,
 621                 struct switchtec_ioctl_flash_part_info *info)
 622 {
 623         struct flash_info_regs_gen3 __iomem *fi =
 624                 &stdev->mmio_flash_info->gen3;
 625         struct sys_info_regs_gen3 __iomem *si = &stdev->mmio_sys_info->gen3;
 626         u32 active_addr = -1;
 627
 628         switch (info->flash_partition) {
 629         case SWITCHTEC_IOCTL_PART_CFG0:
 630                 active_addr = ioread32(&fi->active_cfg);
 631                 set_fw_info_part(info, &fi->cfg0);
 632                 if (ioread16(&si->cfg_running) == SWITCHTEC_GEN3_CFG0_RUNNING)
 633                         info->active |= SWITCHTEC_IOCTL_PART_RUNNING;
 634                 break;
 635         case SWITCHTEC_IOCTL_PART_CFG1:
 636                 active_addr = ioread32(&fi->active_cfg);
 637                 set_fw_info_part(info, &fi->cfg1);
 638                 if (ioread16(&si->cfg_running) == SWITCHTEC_GEN3_CFG1_RUNNING)
 639                         info->active |= SWITCHTEC_IOCTL_PART_RUNNING;
 640                 break;
 641         case SWITCHTEC_IOCTL_PART_IMG0:
 642                 active_addr = ioread32(&fi->active_img);
 643                 set_fw_info_part(info, &fi->img0);
 644                 if (ioread16(&si->img_running) == SWITCHTEC_GEN3_IMG0_RUNNING)
 645                         info->active |= SWITCHTEC_IOCTL_PART_RUNNING;
 646                 break;
 647         case SWITCHTEC_IOCTL_PART_IMG1:
 648                 active_addr = ioread32(&fi->active_img);
 649                 set_fw_info_part(info, &fi->img1);
 650                 if (ioread16(&si->img_running) == SWITCHTEC_GEN3_IMG1_RUNNING)
 651                         info->active |= SWITCHTEC_IOCTL_PART_RUNNING;
 652                 break;
 653         case SWITCHTEC_IOCTL_PART_NVLOG:
 654                 set_fw_info_part(info, &fi->nvlog);
 655                 break;
 656         case SWITCHTEC_IOCTL_PART_VENDOR0:
 657                 set_fw_info_part(info, &fi->vendor[0]);
 658                 break;
 659         case SWITCHTEC_IOCTL_PART_VENDOR1:
 660                 set_fw_info_part(info, &fi->vendor[1]);
 661                 break;
 662         case SWITCHTEC_IOCTL_PART_VENDOR2:
 663                 set_fw_info_part(info, &fi->vendor[2]);
 664                 break;
 665         case SWITCHTEC_IOCTL_PART_VENDOR3:
 666                 set_fw_info_part(info, &fi->vendor[3]);
 667                 break;
 668         case SWITCHTEC_IOCTL_PART_VENDOR4:
 669                 set_fw_info_part(info, &fi->vendor[4]);
 670                 break;
 671         case SWITCHTEC_IOCTL_PART_VENDOR5:
 672                 set_fw_info_part(info, &fi->vendor[5]);
 673                 break;
 674         case SWITCHTEC_IOCTL_PART_VENDOR6:
 675                 set_fw_info_part(info, &fi->vendor[6]);
 676                 break;
 677         case SWITCHTEC_IOCTL_PART_VENDOR7:
 678                 set_fw_info_part(info, &fi->vendor[7]);
 679                 break;
 680         default:
 681                 return -EINVAL;
 682         }
 683
 684         if (info->address == active_addr)
 685                 info->active |= SWITCHTEC_IOCTL_PART_ACTIVE;
 686
 687         return 0;
 688 }
 689
 690 static int ioctl_flash_part_info(struct switchtec_dev *stdev,
 691                 struct switchtec_ioctl_flash_part_info __user *uinfo)
 692 {
 693         int ret;
 694         struct switchtec_ioctl_flash_part_info info = {0};
 695
 696         if (copy_from_user(&info, uinfo, sizeof(info)))
 697                 return -EFAULT;
 698
 699         if (stdev->gen == SWITCHTEC_GEN3) {
 700                 ret = flash_part_info_gen3(stdev, &info);
 701                 if (ret)
 702                         return ret;
 703         } else {
 704                 return -ENOTSUPP;
 705         }
 706
 707         if (copy_to_user(uinfo, &info, sizeof(info)))
 708                 return -EFAULT;
 709
 710         return 0;
 711 }
 712
 713 static int ioctl_event_summary(struct switchtec_dev *stdev,
 714         struct switchtec_user *stuser,
 715         struct switchtec_ioctl_event_summary __user *usum,
 716         size_t size)
 717 {
 718         struct switchtec_ioctl_event_summary *s;
 719         int i;
 720         u32 reg;
 721         int ret = 0;
 722
 723         s = kzalloc(sizeof(*s), GFP_KERNEL);
 724         if (!s)
 725                 return -ENOMEM;
 726
 727         s->global = ioread32(&stdev->mmio_sw_event->global_summary);
 728         s->part_bitmap = ioread64(&stdev->mmio_sw_event->part_event_bitmap);
 729         s->local_part = ioread32(&stdev->mmio_part_cfg->part_event_summary);
 730
 731         for (i = 0; i < stdev->partition_count; i++) {
 732                 reg = ioread32(&stdev->mmio_part_cfg_all[i].part_event_summary);
 733                 s->part[i] = reg;
 734         }
 735
 736         for (i = 0; i < stdev->pff_csr_count; i++) {
 737                 reg = ioread32(&stdev->mmio_pff_csr[i].pff_event_summary);
 738                 s->pff[i] = reg;
 739         }
 740
 741         if (copy_to_user(usum, s, size)) {
 742                 ret = -EFAULT;
 743                 goto error_case;
 744         }
 745
 746         stuser->event_cnt = atomic_read(&stdev->event_cnt);
 747
 748 error_case:
 749         kfree(s);
 750         return ret;
 751 }
 752
 753 static u32 __iomem *global_ev_reg(struct switchtec_dev *stdev,
 754                                   size_t offset, int index)
 755 {
 756         return (void __iomem *)stdev->mmio_sw_event + offset;
 757 }
 758
 759 static u32 __iomem *part_ev_reg(struct switchtec_dev *stdev,
 760                                 size_t offset, int index)
 761 {
 762         return (void __iomem *)&stdev->mmio_part_cfg_all[index] + offset;
 763 }
 764
 765 static u32 __iomem *pff_ev_reg(struct switchtec_dev *stdev,
 766                                size_t offset, int index)
 767 {
 768         return (void __iomem *)&stdev->mmio_pff_csr[index] + offset;
 769 }
 770
 771 #define EV_GLB(i, r)[i] = {offsetof(struct sw_event_regs, r), global_ev_reg}
 772 #define EV_PAR(i, r)[i] = {offsetof(struct part_cfg_regs, r), part_ev_reg}
 773 #define EV_PFF(i, r)[i] = {offsetof(struct pff_csr_regs, r), pff_ev_reg}
 774
 775 static const struct event_reg {
 776         size_t offset;
 777         u32 __iomem *(*map_reg)(struct switchtec_dev *stdev,
 778                                 size_t offset, int index);
 779 } event_regs[] = {
 780         EV_GLB(SWITCHTEC_IOCTL_EVENT_STACK_ERROR, stack_error_event_hdr),
 781         EV_GLB(SWITCHTEC_IOCTL_EVENT_PPU_ERROR, ppu_error_event_hdr),
 782         EV_GLB(SWITCHTEC_IOCTL_EVENT_ISP_ERROR, isp_error_event_hdr),
 783         EV_GLB(SWITCHTEC_IOCTL_EVENT_SYS_RESET, sys_reset_event_hdr),
 784         EV_GLB(SWITCHTEC_IOCTL_EVENT_FW_EXC, fw_exception_hdr),
 785         EV_GLB(SWITCHTEC_IOCTL_EVENT_FW_NMI, fw_nmi_hdr),
 786         EV_GLB(SWITCHTEC_IOCTL_EVENT_FW_NON_FATAL, fw_non_fatal_hdr),
 787         EV_GLB(SWITCHTEC_IOCTL_EVENT_FW_FATAL, fw_fatal_hdr),
 788         EV_GLB(SWITCHTEC_IOCTL_EVENT_TWI_MRPC_COMP, twi_mrpc_comp_hdr),
 789         EV_GLB(SWITCHTEC_IOCTL_EVENT_TWI_MRPC_COMP_ASYNC,
 790                twi_mrpc_comp_async_hdr),
 791         EV_GLB(SWITCHTEC_IOCTL_EVENT_CLI_MRPC_COMP, cli_mrpc_comp_hdr),
 792         EV_GLB(SWITCHTEC_IOCTL_EVENT_CLI_MRPC_COMP_ASYNC,
 793                cli_mrpc_comp_async_hdr),
 794         EV_GLB(SWITCHTEC_IOCTL_EVENT_GPIO_INT, gpio_interrupt_hdr),
 795         EV_GLB(SWITCHTEC_IOCTL_EVENT_GFMS, gfms_event_hdr),
 796         EV_PAR(SWITCHTEC_IOCTL_EVENT_PART_RESET, part_reset_hdr),
 797         EV_PAR(SWITCHTEC_IOCTL_EVENT_MRPC_COMP, mrpc_comp_hdr),
 798         EV_PAR(SWITCHTEC_IOCTL_EVENT_MRPC_COMP_ASYNC, mrpc_comp_async_hdr),
 799         EV_PAR(SWITCHTEC_IOCTL_EVENT_DYN_PART_BIND_COMP, dyn_binding_hdr),
 800         EV_PAR(SWITCHTEC_IOCTL_EVENT_INTERCOMM_REQ_NOTIFY,
 801                intercomm_notify_hdr),
 802         EV_PFF(SWITCHTEC_IOCTL_EVENT_AER_IN_P2P, aer_in_p2p_hdr),
 803         EV_PFF(SWITCHTEC_IOCTL_EVENT_AER_IN_VEP, aer_in_vep_hdr),
 804         EV_PFF(SWITCHTEC_IOCTL_EVENT_DPC, dpc_hdr),
 805         EV_PFF(SWITCHTEC_IOCTL_EVENT_CTS, cts_hdr),
 806         EV_PFF(SWITCHTEC_IOCTL_EVENT_UEC, uec_hdr),
 807         EV_PFF(SWITCHTEC_IOCTL_EVENT_HOTPLUG, hotplug_hdr),
 808         EV_PFF(SWITCHTEC_IOCTL_EVENT_IER, ier_hdr),
 809         EV_PFF(SWITCHTEC_IOCTL_EVENT_THRESH, threshold_hdr),
 810         EV_PFF(SWITCHTEC_IOCTL_EVENT_POWER_MGMT, power_mgmt_hdr),
 811         EV_PFF(SWITCHTEC_IOCTL_EVENT_TLP_THROTTLING, tlp_throttling_hdr),
 812         EV_PFF(SWITCHTEC_IOCTL_EVENT_FORCE_SPEED, force_speed_hdr),
 813         EV_PFF(SWITCHTEC_IOCTL_EVENT_CREDIT_TIMEOUT, credit_timeout_hdr),
 814         EV_PFF(SWITCHTEC_IOCTL_EVENT_LINK_STATE, link_state_hdr),
 815 };
 816
 817 static u32 __iomem *event_hdr_addr(struct switchtec_dev *stdev,
 818                                    int event_id, int index)
 819 {
 820         size_t off;
 821
 822         if (event_id < 0 || event_id >= SWITCHTEC_IOCTL_MAX_EVENTS)
 823                 return ERR_PTR(-EINVAL);
 824
 825         off = event_regs[event_id].offset;
 826
 827         if (event_regs[event_id].map_reg == part_ev_reg) {
 828                 if (index == SWITCHTEC_IOCTL_EVENT_LOCAL_PART_IDX)
 829                         index = stdev->partition;
 830                 else if (index < 0 || index >= stdev->partition_count)
 831                         return ERR_PTR(-EINVAL);
 832         } else if (event_regs[event_id].map_reg == pff_ev_reg) {
 833                 if (index < 0 || index >= stdev->pff_csr_count)
 834                         return ERR_PTR(-EINVAL);
 835         }
 836
 837         return event_regs[event_id].map_reg(stdev, off, index);
 838 }
 839
 840 static int event_ctl(struct switchtec_dev *stdev,
 841                      struct switchtec_ioctl_event_ctl *ctl)
 842 {
 843         int i;
 844         u32 __iomem *reg;
 845         u32 hdr;
 846
 847         reg = event_hdr_addr(stdev, ctl->event_id, ctl->index);
 848         if (IS_ERR(reg))
 849                 return PTR_ERR(reg);
 850
 851         hdr = ioread32(reg);
 852         for (i = 0; i < ARRAY_SIZE(ctl->data); i++)
 853                 ctl->data[i] = ioread32(&reg[i + 1]);
 854
 855         ctl->occurred = hdr & SWITCHTEC_EVENT_OCCURRED;
 856         ctl->count = (hdr >> 5) & 0xFF;
 857
 858         if (!(ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_CLEAR))
 859                 hdr &= ~SWITCHTEC_EVENT_CLEAR;
 860         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_EN_POLL)
 861                 hdr |= SWITCHTEC_EVENT_EN_IRQ;
 862         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_DIS_POLL)
 863                 hdr &= ~SWITCHTEC_EVENT_EN_IRQ;
 864         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_EN_LOG)
 865                 hdr |= SWITCHTEC_EVENT_EN_LOG;
 866         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_DIS_LOG)
 867                 hdr &= ~SWITCHTEC_EVENT_EN_LOG;
 868         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_EN_CLI)
 869                 hdr |= SWITCHTEC_EVENT_EN_CLI;
 870         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_DIS_CLI)
 871                 hdr &= ~SWITCHTEC_EVENT_EN_CLI;
 872         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_EN_FATAL)
 873                 hdr |= SWITCHTEC_EVENT_FATAL;
 874         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_DIS_FATAL)
 875                 hdr &= ~SWITCHTEC_EVENT_FATAL;
 876
 877         if (ctl->flags)
 878                 iowrite32(hdr, reg);
 879
 880         ctl->flags = 0;
 881         if (hdr & SWITCHTEC_EVENT_EN_IRQ)
 882                 ctl->flags |= SWITCHTEC_IOCTL_EVENT_FLAG_EN_POLL;
 883         if (hdr & SWITCHTEC_EVENT_EN_LOG)
 884                 ctl->flags |= SWITCHTEC_IOCTL_EVENT_FLAG_EN_LOG;
 885         if (hdr & SWITCHTEC_EVENT_EN_CLI)
 886                 ctl->flags |= SWITCHTEC_IOCTL_EVENT_FLAG_EN_CLI;
 887         if (hdr & SWITCHTEC_EVENT_FATAL)
 888                 ctl->flags |= SWITCHTEC_IOCTL_EVENT_FLAG_EN_FATAL;
 889
 890         return 0;
 891 }
 892
 893 static int ioctl_event_ctl(struct switchtec_dev *stdev,
 894         struct switchtec_ioctl_event_ctl __user *uctl)
 895 {
 896         int ret;
 897         int nr_idxs;
 898         unsigned int event_flags;
 899         struct switchtec_ioctl_event_ctl ctl;
 900
 901         if (copy_from_user(&ctl, uctl, sizeof(ctl)))
 902                 return -EFAULT;
 903
 904         if (ctl.event_id >= SWITCHTEC_IOCTL_MAX_EVENTS)
 905                 return -EINVAL;
 906
 907         if (ctl.flags & SWITCHTEC_IOCTL_EVENT_FLAG_UNUSED)
 908                 return -EINVAL;
 909
 910         if (ctl.index == SWITCHTEC_IOCTL_EVENT_IDX_ALL) {
 911                 if (event_regs[ctl.event_id].map_reg == global_ev_reg)
 912                         nr_idxs = 1;
 913                 else if (event_regs[ctl.event_id].map_reg == part_ev_reg)
 914                         nr_idxs = stdev->partition_count;
 915                 else if (event_regs[ctl.event_id].map_reg == pff_ev_reg)
 916                         nr_idxs = stdev->pff_csr_count;
 917                 else
 918                         return -EINVAL;
 919
 920                 event_flags = ctl.flags;
 921                 for (ctl.index = 0; ctl.index < nr_idxs; ctl.index++) {
 922                         ctl.flags = event_flags;
 923                         ret = event_ctl(stdev, &ctl);
 924                         if (ret < 0)
 925                                 return ret;
 926                 }
 927         } else {
 928                 ret = event_ctl(stdev, &ctl);
 929                 if (ret < 0)
 930                         return ret;
 931         }
 932
 933         if (copy_to_user(uctl, &ctl, sizeof(ctl)))
 934                 return -EFAULT;
 935
 936         return 0;
 937 }
 938
 939 static int ioctl_pff_to_port(struct switchtec_dev *stdev,
 940                              struct switchtec_ioctl_pff_port *up)
 941 {
 942         int i, part;
 943         u32 reg;
 944         struct part_cfg_regs *pcfg;
 945         struct switchtec_ioctl_pff_port p;
 946
 947         if (copy_from_user(&p, up, sizeof(p)))
 948                 return -EFAULT;
 949
 950         p.port = -1;
 951         for (part = 0; part < stdev->partition_count; part++) {
 952                 pcfg = &stdev->mmio_part_cfg_all[part];
 953                 p.partition = part;
 954
 955                 reg = ioread32(&pcfg->usp_pff_inst_id);
 956                 if (reg == p.pff) {
 957                         p.port = 0;
 958                         break;
 959                 }
 960
 961                 reg = ioread32(&pcfg->vep_pff_inst_id);
 962                 if (reg == p.pff) {
 963                         p.port = SWITCHTEC_IOCTL_PFF_VEP;
 964                         break;
 965                 }
 966
 967                 for (i = 0; i < ARRAY_SIZE(pcfg->dsp_pff_inst_id); i++) {
 968                         reg = ioread32(&pcfg->dsp_pff_inst_id[i]);
 969                         if (reg != p.pff)
 970                                 continue;
 971
 972                         p.port = i + 1;
 973                         break;
 974                 }
 975
 976                 if (p.port != -1)
 977                         break;
 978         }
 979
 980         if (copy_to_user(up, &p, sizeof(p)))
 981                 return -EFAULT;
 982
 983         return 0;
 984 }
 985
 986 static int ioctl_port_to_pff(struct switchtec_dev *stdev,
 987                              struct switchtec_ioctl_pff_port *up)
 988 {
 989         struct switchtec_ioctl_pff_port p;
 990         struct part_cfg_regs *pcfg;
 991
 992         if (copy_from_user(&p, up, sizeof(p)))
 993                 return -EFAULT;
 994
 995         if (p.partition == SWITCHTEC_IOCTL_EVENT_LOCAL_PART_IDX)
 996                 pcfg = stdev->mmio_part_cfg;
 997         else if (p.partition < stdev->partition_count)
 998                 pcfg = &stdev->mmio_part_cfg_all[p.partition];
 999         else
1000                 return -EINVAL;
1001
1002         switch (p.port) {
1003         case 0:
1004                 p.pff = ioread32(&pcfg->usp_pff_inst_id);
1005                 break;
1006         case SWITCHTEC_IOCTL_PFF_VEP:
1007                 p.pff = ioread32(&pcfg->vep_pff_inst_id);
1008                 break;
1009         default:
1010                 if (p.port > ARRAY_SIZE(pcfg->dsp_pff_inst_id))
1011                         return -EINVAL;
1012                 p.port = array_index_nospec(p.port,
1013                                         ARRAY_SIZE(pcfg->dsp_pff_inst_id) + 1);
1014                 p.pff = ioread32(&pcfg->dsp_pff_inst_id[p.port - 1]);
1015                 break;
1016         }
1017
1018         if (copy_to_user(up, &p, sizeof(p)))
1019                 return -EFAULT;
1020
1021         return 0;
1022 }
1023
1024 static long switchtec_dev_ioctl(struct file *filp, unsigned int cmd,
1025                                 unsigned long arg)
1026 {
1027         struct switchtec_user *stuser = filp->private_data;
1028         struct switchtec_dev *stdev = stuser->stdev;
1029         int rc;
1030         void __user *argp = (void __user *)arg;
1031
1032         rc = lock_mutex_and_test_alive(stdev);
1033         if (rc)
1034                 return rc;
1035
1036         switch (cmd) {
1037         case SWITCHTEC_IOCTL_FLASH_INFO:
1038                 rc = ioctl_flash_info(stdev, argp);
1039                 break;
1040         case SWITCHTEC_IOCTL_FLASH_PART_INFO:
1041                 rc = ioctl_flash_part_info(stdev, argp);
1042                 break;
1043         case SWITCHTEC_IOCTL_EVENT_SUMMARY_LEGACY:
1044                 rc = ioctl_event_summary(stdev, stuser, argp,
1045                                          sizeof(struct switchtec_ioctl_event_summary_legacy));
1046                 break;
1047         case SWITCHTEC_IOCTL_EVENT_CTL:
1048                 rc = ioctl_event_ctl(stdev, argp);
1049                 break;
1050         case SWITCHTEC_IOCTL_PFF_TO_PORT:
1051                 rc = ioctl_pff_to_port(stdev, argp);
1052                 break;
1053         case SWITCHTEC_IOCTL_PORT_TO_PFF:
1054                 rc = ioctl_port_to_pff(stdev, argp);
1055                 break;
1056         case SWITCHTEC_IOCTL_EVENT_SUMMARY:
1057                 rc = ioctl_event_summary(stdev, stuser, argp,
1058                                          sizeof(struct switchtec_ioctl_event_summary));
1059                 break;
1060         default:
1061                 rc = -ENOTTY;
1062                 break;
1063         }
1064
1065         mutex_unlock(&stdev->mrpc_mutex);
1066         return rc;
1067 }
1068
1069 static const struct file_operations switchtec_fops = {
1070         .owner = THIS_MODULE,
1071         .open = switchtec_dev_open,
1072         .release = switchtec_dev_release,
1073         .write = switchtec_dev_write,
1074         .read = switchtec_dev_read,
1075         .poll = switchtec_dev_poll,
1076         .unlocked_ioctl = switchtec_dev_ioctl,
1077         .compat_ioctl = compat_ptr_ioctl,
1078 };
1079
1080 static void link_event_work(struct work_struct *work)
1081 {
1082         struct switchtec_dev *stdev;
1083
1084         stdev = container_of(work, struct switchtec_dev, link_event_work);
1085
1086         if (stdev->link_notifier)
1087                 stdev->link_notifier(stdev);
1088 }
1089
1090 static void check_link_state_events(struct switchtec_dev *stdev)
1091 {
1092         int idx;
1093         u32 reg;
1094         int count;
1095         int occurred = 0;
1096
1097         for (idx = 0; idx < stdev->pff_csr_count; idx++) {
1098                 reg = ioread32(&stdev->mmio_pff_csr[idx].link_state_hdr);
1099                 dev_dbg(&stdev->dev, "link_state: %d->%08x\n", idx, reg);
1100                 count = (reg >> 5) & 0xFF;
1101
1102                 if (count != stdev->link_event_count[idx]) {
1103                         occurred = 1;
1104                         stdev->link_event_count[idx] = count;
1105                 }
1106         }
1107
1108         if (occurred)
1109                 schedule_work(&stdev->link_event_work);
1110 }
1111
1112 static void enable_link_state_events(struct switchtec_dev *stdev)
1113 {
1114         int idx;
1115
1116         for (idx = 0; idx < stdev->pff_csr_count; idx++) {
1117                 iowrite32(SWITCHTEC_EVENT_CLEAR |
1118                           SWITCHTEC_EVENT_EN_IRQ,
1119                           &stdev->mmio_pff_csr[idx].link_state_hdr);
1120         }
1121 }
1122
1123 static void enable_dma_mrpc(struct switchtec_dev *stdev)
1124 {
1125         writeq(stdev->dma_mrpc_dma_addr, &stdev->mmio_mrpc->dma_addr);
1126         flush_wc_buf(stdev);
1127         iowrite32(SWITCHTEC_DMA_MRPC_EN, &stdev->mmio_mrpc->dma_en);
1128 }
1129
1130 static void stdev_release(struct device *dev)
1131 {
1132         struct switchtec_dev *stdev = to_stdev(dev);
1133
1134         if (stdev->dma_mrpc) {
1135                 iowrite32(0, &stdev->mmio_mrpc->dma_en);
1136                 flush_wc_buf(stdev);
1137                 writeq(0, &stdev->mmio_mrpc->dma_addr);
1138                 dma_free_coherent(&stdev->pdev->dev, sizeof(*stdev->dma_mrpc),
1139                                 stdev->dma_mrpc, stdev->dma_mrpc_dma_addr);
1140         }
1141         kfree(stdev);
1142 }
1143
1144 static void stdev_kill(struct switchtec_dev *stdev)
1145 {
1146         struct switchtec_user *stuser, *tmpuser;
1147
1148         pci_clear_master(stdev->pdev);
1149
1150         cancel_delayed_work_sync(&stdev->mrpc_timeout);
1151
1152         /* Mark the hardware as unavailable and complete all completions */
1153         mutex_lock(&stdev->mrpc_mutex);
1154         stdev->alive = false;
1155
1156         /* Wake up and kill any users waiting on an MRPC request */
1157         list_for_each_entry_safe(stuser, tmpuser, &stdev->mrpc_queue, list) {
1158                 complete_all(&stuser->comp);
1159                 list_del_init(&stuser->list);
1160                 stuser_put(stuser);
1161         }
1162
1163         mutex_unlock(&stdev->mrpc_mutex);
1164
1165         /* Wake up any users waiting on event_wq */
1166         wake_up_interruptible(&stdev->event_wq);
1167 }
1168
1169 static struct switchtec_dev *stdev_create(struct pci_dev *pdev)
1170 {
1171         struct switchtec_dev *stdev;
1172         int minor;
1173         struct device *dev;
1174         struct cdev *cdev;
1175         int rc;
1176
1177         stdev = kzalloc_node(sizeof(*stdev), GFP_KERNEL,
1178                              dev_to_node(&pdev->dev));
1179         if (!stdev)
1180                 return ERR_PTR(-ENOMEM);
1181
1182         stdev->alive = true;
1183         stdev->pdev = pdev;
1184         INIT_LIST_HEAD(&stdev->mrpc_queue);
1185         mutex_init(&stdev->mrpc_mutex);
1186         stdev->mrpc_busy = 0;
1187         INIT_WORK(&stdev->mrpc_work, mrpc_event_work);
1188         INIT_DELAYED_WORK(&stdev->mrpc_timeout, mrpc_timeout_work);
1189         INIT_WORK(&stdev->link_event_work, link_event_work);
1190         init_waitqueue_head(&stdev->event_wq);
1191         atomic_set(&stdev->event_cnt, 0);
1192
1193         dev = &stdev->dev;
1194         device_initialize(dev);
1195         dev->class = switchtec_class;
1196         dev->parent = &pdev->dev;
1197         dev->groups = switchtec_device_groups;
1198         dev->release = stdev_release;
1199
1200         minor = ida_simple_get(&switchtec_minor_ida, 0, 0,
1201                                GFP_KERNEL);
1202         if (minor < 0) {
1203                 rc = minor;
1204                 goto err_put;
1205         }
1206
1207         dev->devt = MKDEV(MAJOR(switchtec_devt), minor);
1208         dev_set_name(dev, "switchtec%d", minor);
1209
1210         cdev = &stdev->cdev;
1211         cdev_init(cdev, &switchtec_fops);
1212         cdev->owner = THIS_MODULE;
1213
1214         return stdev;
1215
1216 err_put:
1217         put_device(&stdev->dev);
1218         return ERR_PTR(rc);
1219 }
1220
1221 static int mask_event(struct switchtec_dev *stdev, int eid, int idx)
1222 {
1223         size_t off = event_regs[eid].offset;
1224         u32 __iomem *hdr_reg;
1225         u32 hdr;
1226
1227         hdr_reg = event_regs[eid].map_reg(stdev, off, idx);
1228         hdr = ioread32(hdr_reg);
1229
1230         if (!(hdr & SWITCHTEC_EVENT_OCCURRED && hdr & SWITCHTEC_EVENT_EN_IRQ))
1231                 return 0;
1232
1233         dev_dbg(&stdev->dev, "%s: %d %d %x\n", __func__, eid, idx, hdr);
1234         hdr &= ~(SWITCHTEC_EVENT_EN_IRQ | SWITCHTEC_EVENT_OCCURRED);
1235         iowrite32(hdr, hdr_reg);
1236
1237         return 1;
1238 }
1239
1240 static int mask_all_events(struct switchtec_dev *stdev, int eid)
1241 {
1242         int idx;
1243         int count = 0;
1244
1245         if (event_regs[eid].map_reg == part_ev_reg) {
1246                 for (idx = 0; idx < stdev->partition_count; idx++)
1247                         count += mask_event(stdev, eid, idx);
1248         } else if (event_regs[eid].map_reg == pff_ev_reg) {
1249                 for (idx = 0; idx < stdev->pff_csr_count; idx++) {
1250                         if (!stdev->pff_local[idx])
1251                                 continue;
1252
1253                         count += mask_event(stdev, eid, idx);
1254                 }
1255         } else {
1256                 count += mask_event(stdev, eid, 0);
1257         }
1258
1259         return count;
1260 }
1261
1262 static irqreturn_t switchtec_event_isr(int irq, void *dev)
1263 {
1264         struct switchtec_dev *stdev = dev;
1265         u32 reg;
1266         irqreturn_t ret = IRQ_NONE;
1267         int eid, event_count = 0;
1268
1269         reg = ioread32(&stdev->mmio_part_cfg->mrpc_comp_hdr);
1270         if (reg & SWITCHTEC_EVENT_OCCURRED) {
1271                 dev_dbg(&stdev->dev, "%s: mrpc comp\n", __func__);
1272                 ret = IRQ_HANDLED;
1273                 schedule_work(&stdev->mrpc_work);
1274                 iowrite32(reg, &stdev->mmio_part_cfg->mrpc_comp_hdr);
1275         }
1276
1277         check_link_state_events(stdev);
1278
1279         for (eid = 0; eid < SWITCHTEC_IOCTL_MAX_EVENTS; eid++) {
1280                 if (eid == SWITCHTEC_IOCTL_EVENT_LINK_STATE ||
1281                     eid == SWITCHTEC_IOCTL_EVENT_MRPC_COMP)
1282                         continue;
1283
1284                 event_count += mask_all_events(stdev, eid);
1285         }
1286
1287         if (event_count) {
1288                 atomic_inc(&stdev->event_cnt);
1289                 wake_up_interruptible(&stdev->event_wq);
1290                 dev_dbg(&stdev->dev, "%s: %d events\n", __func__,
1291                         event_count);
1292                 return IRQ_HANDLED;
1293         }
1294
1295         return ret;
1296 }
1297
1298
1299 static irqreturn_t switchtec_dma_mrpc_isr(int irq, void *dev)
1300 {
1301         struct switchtec_dev *stdev = dev;
1302         irqreturn_t ret = IRQ_NONE;
1303
1304         iowrite32(SWITCHTEC_EVENT_CLEAR |
1305                   SWITCHTEC_EVENT_EN_IRQ,
1306                   &stdev->mmio_part_cfg->mrpc_comp_hdr);
1307         schedule_work(&stdev->mrpc_work);
1308
1309         ret = IRQ_HANDLED;
1310         return ret;
1311 }
1312
1313 static int switchtec_init_isr(struct switchtec_dev *stdev)
1314 {
1315         int nvecs;
1316         int event_irq;
1317         int dma_mrpc_irq;
1318         int rc;
1319
1320         if (nirqs < 4)
1321                 nirqs = 4;
1322
1323         nvecs = pci_alloc_irq_vectors(stdev->pdev, 1, nirqs,
1324                                       PCI_IRQ_MSIX | PCI_IRQ_MSI |
1325                                       PCI_IRQ_VIRTUAL);
1326         if (nvecs < 0)
1327                 return nvecs;
1328
1329         event_irq = ioread16(&stdev->mmio_part_cfg->vep_vector_number);
1330         if (event_irq < 0 || event_irq >= nvecs)
1331                 return -EFAULT;
1332
1333         event_irq = pci_irq_vector(stdev->pdev, event_irq);
1334         if (event_irq < 0)
1335                 return event_irq;
1336
1337         rc = devm_request_irq(&stdev->pdev->dev, event_irq,
1338                                 switchtec_event_isr, 0,
1339                                 KBUILD_MODNAME, stdev);
1340
1341         if (rc)
1342                 return rc;
1343
1344         if (!stdev->dma_mrpc)
1345                 return rc;
1346
1347         dma_mrpc_irq = ioread32(&stdev->mmio_mrpc->dma_vector);
1348         if (dma_mrpc_irq < 0 || dma_mrpc_irq >= nvecs)
1349                 return -EFAULT;
1350
1351         dma_mrpc_irq  = pci_irq_vector(stdev->pdev, dma_mrpc_irq);
1352         if (dma_mrpc_irq < 0)
1353                 return dma_mrpc_irq;
1354
1355         rc = devm_request_irq(&stdev->pdev->dev, dma_mrpc_irq,
1356                                 switchtec_dma_mrpc_isr, 0,
1357                                 KBUILD_MODNAME, stdev);
1358
1359         return rc;
1360 }
1361
1362 static void init_pff(struct switchtec_dev *stdev)
1363 {
1364         int i;
1365         u32 reg;
1366         struct part_cfg_regs *pcfg = stdev->mmio_part_cfg;
1367
1368         for (i = 0; i < SWITCHTEC_MAX_PFF_CSR; i++) {
1369                 reg = ioread16(&stdev->mmio_pff_csr[i].vendor_id);
1370                 if (reg != PCI_VENDOR_ID_MICROSEMI)
1371                         break;
1372         }
1373
1374         stdev->pff_csr_count = i;
1375
1376         reg = ioread32(&pcfg->usp_pff_inst_id);
1377         if (reg < stdev->pff_csr_count)
1378                 stdev->pff_local[reg] = 1;
1379
1380         reg = ioread32(&pcfg->vep_pff_inst_id);
1381         if (reg < stdev->pff_csr_count)
1382                 stdev->pff_local[reg] = 1;
1383
1384         for (i = 0; i < ARRAY_SIZE(pcfg->dsp_pff_inst_id); i++) {
1385                 reg = ioread32(&pcfg->dsp_pff_inst_id[i]);
1386                 if (reg < stdev->pff_csr_count)
1387                         stdev->pff_local[reg] = 1;
1388         }
1389 }
1390
1391 static int switchtec_init_pci(struct switchtec_dev *stdev,
1392                               struct pci_dev *pdev)
1393 {
1394         int rc;
1395         void __iomem *map;
1396         unsigned long res_start, res_len;
1397         u32 __iomem *part_id;
1398
1399         rc = pcim_enable_device(pdev);
1400         if (rc)
1401                 return rc;
1402
1403         rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1404         if (rc)
1405                 return rc;
1406
1407         pci_set_master(pdev);
1408
1409         res_start = pci_resource_start(pdev, 0);
1410         res_len = pci_resource_len(pdev, 0);
1411
1412         if (!devm_request_mem_region(&pdev->dev, res_start,
1413                                      res_len, KBUILD_MODNAME))
1414                 return -EBUSY;
1415
1416         stdev->mmio_mrpc = devm_ioremap_wc(&pdev->dev, res_start,
1417                                            SWITCHTEC_GAS_TOP_CFG_OFFSET);
1418         if (!stdev->mmio_mrpc)
1419                 return -ENOMEM;
1420
1421         map = devm_ioremap(&pdev->dev,
1422                            res_start + SWITCHTEC_GAS_TOP_CFG_OFFSET,
1423                            res_len - SWITCHTEC_GAS_TOP_CFG_OFFSET);
1424         if (!map)
1425                 return -ENOMEM;
1426
1427         stdev->mmio = map - SWITCHTEC_GAS_TOP_CFG_OFFSET;
1428         stdev->mmio_sw_event = stdev->mmio + SWITCHTEC_GAS_SW_EVENT_OFFSET;
1429         stdev->mmio_sys_info = stdev->mmio + SWITCHTEC_GAS_SYS_INFO_OFFSET;
1430         stdev->mmio_flash_info = stdev->mmio + SWITCHTEC_GAS_FLASH_INFO_OFFSET;
1431         stdev->mmio_ntb = stdev->mmio + SWITCHTEC_GAS_NTB_OFFSET;
1432
1433         if (stdev->gen == SWITCHTEC_GEN3)
1434                 part_id = &stdev->mmio_sys_info->gen3.partition_id;
1435         else if (stdev->gen == SWITCHTEC_GEN4)
1436                 part_id = &stdev->mmio_sys_info->gen4.partition_id;
1437         else
1438                 return -ENOTSUPP;
1439
1440         stdev->partition = ioread8(part_id);
1441         stdev->partition_count = ioread8(&stdev->mmio_ntb->partition_count);
1442         stdev->mmio_part_cfg_all = stdev->mmio + SWITCHTEC_GAS_PART_CFG_OFFSET;
1443         stdev->mmio_part_cfg = &stdev->mmio_part_cfg_all[stdev->partition];
1444         stdev->mmio_pff_csr = stdev->mmio + SWITCHTEC_GAS_PFF_CSR_OFFSET;
1445
1446         if (stdev->partition_count < 1)
1447                 stdev->partition_count = 1;
1448
1449         init_pff(stdev);
1450
1451         pci_set_drvdata(pdev, stdev);
1452
1453         if (!use_dma_mrpc)
1454                 return 0;
1455
1456         if (ioread32(&stdev->mmio_mrpc->dma_ver) == 0)
1457                 return 0;
1458
1459         stdev->dma_mrpc = dma_alloc_coherent(&stdev->pdev->dev,
1460                                              sizeof(*stdev->dma_mrpc),
1461                                              &stdev->dma_mrpc_dma_addr,
1462                                              GFP_KERNEL);
1463         if (stdev->dma_mrpc == NULL)
1464                 return -ENOMEM;
1465
1466         return 0;
1467 }
1468
1469 static int switchtec_pci_probe(struct pci_dev *pdev,
1470                                const struct pci_device_id *id)
1471 {
1472         struct switchtec_dev *stdev;
1473         int rc;
1474
1475         if (pdev->class == (PCI_CLASS_BRIDGE_OTHER << 8))
1476                 request_module_nowait("ntb_hw_switchtec");
1477
1478         stdev = stdev_create(pdev);
1479         if (IS_ERR(stdev))
1480                 return PTR_ERR(stdev);
1481
1482         stdev->gen = id->driver_data;
1483
1484         rc = switchtec_init_pci(stdev, pdev);
1485         if (rc)
1486                 goto err_put;
1487
1488         rc = switchtec_init_isr(stdev);
1489         if (rc) {
1490                 dev_err(&stdev->dev, "failed to init isr.\n");
1491                 goto err_put;
1492         }
1493
1494         iowrite32(SWITCHTEC_EVENT_CLEAR |
1495                   SWITCHTEC_EVENT_EN_IRQ,
1496                   &stdev->mmio_part_cfg->mrpc_comp_hdr);
1497         enable_link_state_events(stdev);
1498
1499         if (stdev->dma_mrpc)
1500                 enable_dma_mrpc(stdev);
1501
1502         rc = cdev_device_add(&stdev->cdev, &stdev->dev);
1503         if (rc)
1504                 goto err_devadd;
1505
1506         dev_info(&stdev->dev, "Management device registered.\n");
1507
1508         return 0;
1509
1510 err_devadd:
1511         stdev_kill(stdev);
1512 err_put:
1513         ida_simple_remove(&switchtec_minor_ida, MINOR(stdev->dev.devt));
1514         put_device(&stdev->dev);
1515         return rc;
1516 }
1517
1518 static void switchtec_pci_remove(struct pci_dev *pdev)
1519 {
1520         struct switchtec_dev *stdev = pci_get_drvdata(pdev);
1521
1522         pci_set_drvdata(pdev, NULL);
1523
1524         cdev_device_del(&stdev->cdev, &stdev->dev);
1525         ida_simple_remove(&switchtec_minor_ida, MINOR(stdev->dev.devt));
1526         dev_info(&stdev->dev, "unregistered.\n");
1527         stdev_kill(stdev);
1528         put_device(&stdev->dev);
1529 }
1530
1531 #define SWITCHTEC_PCI_DEVICE(device_id, gen) \
1532         { \
1533                 .vendor     = PCI_VENDOR_ID_MICROSEMI, \
1534                 .device     = device_id, \
1535                 .subvendor  = PCI_ANY_ID, \
1536                 .subdevice  = PCI_ANY_ID, \
1537                 .class      = (PCI_CLASS_MEMORY_OTHER << 8), \
1538                 .class_mask = 0xFFFFFFFF, \
1539                 .driver_data = gen, \
1540         }, \
1541         { \
1542                 .vendor     = PCI_VENDOR_ID_MICROSEMI, \
1543                 .device     = device_id, \
1544                 .subvendor  = PCI_ANY_ID, \
1545                 .subdevice  = PCI_ANY_ID, \
1546                 .class      = (PCI_CLASS_BRIDGE_OTHER << 8), \
1547                 .class_mask = 0xFFFFFFFF, \
1548                 .driver_data = gen, \
1549         }
1550
1551 static const struct pci_device_id switchtec_pci_tbl[] = {
1552         SWITCHTEC_PCI_DEVICE(0x8531, SWITCHTEC_GEN3),  //PFX 24xG3
1553         SWITCHTEC_PCI_DEVICE(0x8532, SWITCHTEC_GEN3),  //PFX 32xG3
1554         SWITCHTEC_PCI_DEVICE(0x8533, SWITCHTEC_GEN3),  //PFX 48xG3
1555         SWITCHTEC_PCI_DEVICE(0x8534, SWITCHTEC_GEN3),  //PFX 64xG3
1556         SWITCHTEC_PCI_DEVICE(0x8535, SWITCHTEC_GEN3),  //PFX 80xG3
1557         SWITCHTEC_PCI_DEVICE(0x8536, SWITCHTEC_GEN3),  //PFX 96xG3
1558         SWITCHTEC_PCI_DEVICE(0x8541, SWITCHTEC_GEN3),  //PSX 24xG3
1559         SWITCHTEC_PCI_DEVICE(0x8542, SWITCHTEC_GEN3),  //PSX 32xG3
1560         SWITCHTEC_PCI_DEVICE(0x8543, SWITCHTEC_GEN3),  //PSX 48xG3
1561         SWITCHTEC_PCI_DEVICE(0x8544, SWITCHTEC_GEN3),  //PSX 64xG3
1562         SWITCHTEC_PCI_DEVICE(0x8545, SWITCHTEC_GEN3),  //PSX 80xG3
1563         SWITCHTEC_PCI_DEVICE(0x8546, SWITCHTEC_GEN3),  //PSX 96xG3
1564         SWITCHTEC_PCI_DEVICE(0x8551, SWITCHTEC_GEN3),  //PAX 24XG3
1565         SWITCHTEC_PCI_DEVICE(0x8552, SWITCHTEC_GEN3),  //PAX 32XG3
1566         SWITCHTEC_PCI_DEVICE(0x8553, SWITCHTEC_GEN3),  //PAX 48XG3
1567         SWITCHTEC_PCI_DEVICE(0x8554, SWITCHTEC_GEN3),  //PAX 64XG3
1568         SWITCHTEC_PCI_DEVICE(0x8555, SWITCHTEC_GEN3),  //PAX 80XG3
1569         SWITCHTEC_PCI_DEVICE(0x8556, SWITCHTEC_GEN3),  //PAX 96XG3
1570         SWITCHTEC_PCI_DEVICE(0x8561, SWITCHTEC_GEN3),  //PFXL 24XG3
1571         SWITCHTEC_PCI_DEVICE(0x8562, SWITCHTEC_GEN3),  //PFXL 32XG3
1572         SWITCHTEC_PCI_DEVICE(0x8563, SWITCHTEC_GEN3),  //PFXL 48XG3
1573         SWITCHTEC_PCI_DEVICE(0x8564, SWITCHTEC_GEN3),  //PFXL 64XG3
1574         SWITCHTEC_PCI_DEVICE(0x8565, SWITCHTEC_GEN3),  //PFXL 80XG3
1575         SWITCHTEC_PCI_DEVICE(0x8566, SWITCHTEC_GEN3),  //PFXL 96XG3
1576         SWITCHTEC_PCI_DEVICE(0x8571, SWITCHTEC_GEN3),  //PFXI 24XG3
1577         SWITCHTEC_PCI_DEVICE(0x8572, SWITCHTEC_GEN3),  //PFXI 32XG3
1578         SWITCHTEC_PCI_DEVICE(0x8573, SWITCHTEC_GEN3),  //PFXI 48XG3
1579         SWITCHTEC_PCI_DEVICE(0x8574, SWITCHTEC_GEN3),  //PFXI 64XG3
1580         SWITCHTEC_PCI_DEVICE(0x8575, SWITCHTEC_GEN3),  //PFXI 80XG3
1581         SWITCHTEC_PCI_DEVICE(0x8576, SWITCHTEC_GEN3),  //PFXI 96XG3
1582         {0}
1583 };
1584 MODULE_DEVICE_TABLE(pci, switchtec_pci_tbl);
1585
1586 static struct pci_driver switchtec_pci_driver = {
1587         .name           = KBUILD_MODNAME,
1588         .id_table       = switchtec_pci_tbl,
1589         .probe          = switchtec_pci_probe,
1590         .remove         = switchtec_pci_remove,
1591 };
1592
1593 static int __init switchtec_init(void)
1594 {
1595         int rc;
1596
1597         rc = alloc_chrdev_region(&switchtec_devt, 0, max_devices,
1598                                  "switchtec");
1599         if (rc)
1600                 return rc;
1601
1602         switchtec_class = class_create(THIS_MODULE, "switchtec");
1603         if (IS_ERR(switchtec_class)) {
1604                 rc = PTR_ERR(switchtec_class);
1605                 goto err_create_class;
1606         }
1607
1608         rc = pci_register_driver(&switchtec_pci_driver);
1609         if (rc)
1610                 goto err_pci_register;
1611
1612         pr_info(KBUILD_MODNAME ": loaded.\n");
1613
1614         return 0;
1615
1616 err_pci_register:
1617         class_destroy(switchtec_class);
1618
1619 err_create_class:
1620         unregister_chrdev_region(switchtec_devt, max_devices);
1621
1622         return rc;
1623 }
1624 module_init(switchtec_init);
1625
1626 static void __exit switchtec_exit(void)
1627 {
1628         pci_unregister_driver(&switchtec_pci_driver);
1629         class_destroy(switchtec_class);
1630         unregister_chrdev_region(switchtec_devt, max_devices);
1631         ida_destroy(&switchtec_minor_ida);
1632
1633         pr_info(KBUILD_MODNAME ": unloaded.\n");
1634 }
1635 module_exit(switchtec_exit);