drivers/crypto/cavium/zip/zip_main.c

   1 /***********************license start************************************
   2  * Copyright (c) 2003-2017 Cavium, Inc.
   3  * All rights reserved.
   4  *
   5  * License: one of 'Cavium License' or 'GNU General Public License Version 2'
   6  *
   7  * This file is provided under the terms of the Cavium License (see below)
   8  * or under the terms of GNU General Public License, Version 2, as
   9  * published by the Free Software Foundation. When using or redistributing
  10  * this file, you may do so under either license.
  11  *
  12  * Cavium License:  Redistribution and use in source and binary forms, with
  13  * or without modification, are permitted provided that the following
  14  * conditions are met:
  15  *
  16  *  * Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  *
  19  *  * Redistributions in binary form must reproduce the above
  20  *    copyright notice, this list of conditions and the following
  21  *    disclaimer in the documentation and/or other materials provided
  22  *    with the distribution.
  23  *
  24  *  * Neither the name of Cavium Inc. nor the names of its contributors may be
  25  *    used to endorse or promote products derived from this software without
  26  *    specific prior written permission.
  27  *
  28  * This Software, including technical data, may be subject to U.S. export
  29  * control laws, including the U.S. Export Administration Act and its
  30  * associated regulations, and may be subject to export or import
  31  * regulations in other countries.
  32  *
  33  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  34  * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS
  35  * OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH
  36  * RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY
  37  * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT
  38  * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY)
  39  * WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A
  40  * PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET
  41  * ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE
  42  * ENTIRE  RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES
  43  * WITH YOU.
  44  ***********************license end**************************************/
  45
  46 #include "common.h"
  47 #include "zip_crypto.h"
  48
  49 #define DRV_NAME                "ThunderX-ZIP"
  50
  51 static struct zip_device *zip_dev[MAX_ZIP_DEVICES];
  52
  53 static const struct pci_device_id zip_id_table[] = {
  54         { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDERX_ZIP) },
  55         { 0, }
  56 };
  57
  58 void zip_reg_write(u64 val, u64 __iomem *addr)
  59 {
  60         writeq(val, addr);
  61 }
  62
  63 u64 zip_reg_read(u64 __iomem *addr)
  64 {
  65         return readq(addr);
  66 }
  67
  68 /*
  69  * Allocates new ZIP device structure
  70  * Returns zip_device pointer or NULL if cannot allocate memory for zip_device
  71  */
  72 static struct zip_device *zip_alloc_device(struct pci_dev *pdev)
  73 {
  74         struct zip_device *zip = NULL;
  75         int idx;
  76
  77         for (idx = 0; idx < MAX_ZIP_DEVICES; idx++) {
  78                 if (!zip_dev[idx])
  79                         break;
  80         }
  81
  82         /* To ensure that the index is within the limit */
  83         if (idx < MAX_ZIP_DEVICES)
  84                 zip = devm_kzalloc(&pdev->dev, sizeof(*zip), GFP_KERNEL);
  85
  86         if (!zip)
  87                 return NULL;
  88
  89         zip_dev[idx] = zip;
  90         zip->index = idx;
  91         return zip;
  92 }
  93
  94 /**
  95  * zip_get_device - Get ZIP device based on node id of cpu
  96  *
  97  * @node: Node id of the current cpu
  98  * Return: Pointer to Zip device structure
  99  */
 100 struct zip_device *zip_get_device(int node)
 101 {
 102         if ((node < MAX_ZIP_DEVICES) && (node >= 0))
 103                 return zip_dev[node];
 104
 105         zip_err("ZIP device not found for node id %d\n", node);
 106         return NULL;
 107 }
 108
 109 /**
 110  * zip_get_node_id - Get the node id of the current cpu
 111  *
 112  * Return: Node id of the current cpu
 113  */
 114 int zip_get_node_id(void)
 115 {
 116         return cpu_to_node(smp_processor_id());
 117 }
 118
 119 /* Initializes the ZIP h/w sub-system */
 120 static int zip_init_hw(struct zip_device *zip)
 121 {
 122         union zip_cmd_ctl    cmd_ctl;
 123         union zip_constants  constants;
 124         union zip_que_ena    que_ena;
 125         union zip_quex_map   que_map;
 126         union zip_que_pri    que_pri;
 127
 128         union zip_quex_sbuf_addr que_sbuf_addr;
 129         union zip_quex_sbuf_ctl  que_sbuf_ctl;
 130
 131         int q = 0;
 132
 133         /* Enable the ZIP Engine(Core) Clock */
 134         cmd_ctl.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CMD_CTL);
 135         cmd_ctl.s.forceclk = 1;
 136         zip_reg_write(cmd_ctl.u_reg64 & 0xFF, (zip->reg_base + ZIP_CMD_CTL));
 137
 138         zip_msg("ZIP_CMD_CTL  : 0x%016llx",
 139                 zip_reg_read(zip->reg_base + ZIP_CMD_CTL));
 140
 141         constants.u_reg64 = zip_reg_read(zip->reg_base + ZIP_CONSTANTS);
 142         zip->depth    = constants.s.depth;
 143         zip->onfsize  = constants.s.onfsize;
 144         zip->ctxsize  = constants.s.ctxsize;
 145
 146         zip_msg("depth: 0x%016llx , onfsize : 0x%016llx , ctxsize : 0x%016llx",
 147                 zip->depth, zip->onfsize, zip->ctxsize);
 148
 149         /*
 150          * Program ZIP_QUE(0..7)_SBUF_ADDR and ZIP_QUE(0..7)_SBUF_CTL to
 151          * have the correct buffer pointer and size configured for each
 152          * instruction queue.
 153          */
 154         for (q = 0; q < ZIP_NUM_QUEUES; q++) {
 155                 que_sbuf_ctl.u_reg64 = 0ull;
 156                 que_sbuf_ctl.s.size = (ZIP_CMD_QBUF_SIZE / sizeof(u64));
 157                 que_sbuf_ctl.s.inst_be   = 0;
 158                 que_sbuf_ctl.s.stream_id = 0;
 159                 zip_reg_write(que_sbuf_ctl.u_reg64,
 160                               (zip->reg_base + ZIP_QUEX_SBUF_CTL(q)));
 161
 162                 zip_msg("QUEX_SBUF_CTL[%d]: 0x%016llx", q,
 163                         zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_CTL(q)));
 164         }
 165
 166         for (q = 0; q < ZIP_NUM_QUEUES; q++) {
 167                 memset(&zip->iq[q], 0x0, sizeof(struct zip_iq));
 168
 169                 spin_lock_init(&zip->iq[q].lock);
 170
 171                 if (zip_cmd_qbuf_alloc(zip, q)) {
 172                         while (q != 0) {
 173                                 q--;
 174                                 zip_cmd_qbuf_free(zip, q);
 175                         }
 176                         return -ENOMEM;
 177                 }
 178
 179                 /* Initialize tail ptr to head */
 180                 zip->iq[q].sw_tail = zip->iq[q].sw_head;
 181                 zip->iq[q].hw_tail = zip->iq[q].sw_head;
 182
 183                 /* Write the physical addr to register */
 184                 que_sbuf_addr.u_reg64   = 0ull;
 185                 que_sbuf_addr.s.ptr = (__pa(zip->iq[q].sw_head) >>
 186                                        ZIP_128B_ALIGN);
 187
 188                 zip_msg("QUE[%d]_PTR(PHYS): 0x%016llx", q,
 189                         (u64)que_sbuf_addr.s.ptr);
 190
 191                 zip_reg_write(que_sbuf_addr.u_reg64,
 192                               (zip->reg_base + ZIP_QUEX_SBUF_ADDR(q)));
 193
 194                 zip_msg("QUEX_SBUF_ADDR[%d]: 0x%016llx", q,
 195                         zip_reg_read(zip->reg_base + ZIP_QUEX_SBUF_ADDR(q)));
 196
 197                 zip_dbg("sw_head :0x%lx sw_tail :0x%lx hw_tail :0x%lx",
 198                         zip->iq[q].sw_head, zip->iq[q].sw_tail,
 199                         zip->iq[q].hw_tail);
 200                 zip_dbg("sw_head phy addr : 0x%lx", que_sbuf_addr.s.ptr);
 201         }
 202
 203         /*
 204          * Queue-to-ZIP core mapping
 205          * If a queue is not mapped to a particular core, it is equivalent to
 206          * the ZIP core being disabled.
 207          */
 208         que_ena.u_reg64 = 0x0ull;
 209         /* Enabling queues based on ZIP_NUM_QUEUES */
 210         for (q = 0; q < ZIP_NUM_QUEUES; q++)
 211                 que_ena.s.ena |= (0x1 << q);
 212         zip_reg_write(que_ena.u_reg64, (zip->reg_base + ZIP_QUE_ENA));
 213
 214         zip_msg("QUE_ENA      : 0x%016llx",
 215                 zip_reg_read(zip->reg_base + ZIP_QUE_ENA));
 216
 217         for (q = 0; q < ZIP_NUM_QUEUES; q++) {
 218                 que_map.u_reg64 = 0ull;
 219                 /* Mapping each queue to two ZIP cores */
 220                 que_map.s.zce = 0x3;
 221                 zip_reg_write(que_map.u_reg64,
 222                               (zip->reg_base + ZIP_QUEX_MAP(q)));
 223
 224                 zip_msg("QUE_MAP(%d)   : 0x%016llx", q,
 225                         zip_reg_read(zip->reg_base + ZIP_QUEX_MAP(q)));
 226         }
 227
 228         que_pri.u_reg64 = 0ull;
 229         for (q = 0; q < ZIP_NUM_QUEUES; q++)
 230                 que_pri.s.pri |= (0x1 << q); /* Higher Priority RR */
 231         zip_reg_write(que_pri.u_reg64, (zip->reg_base + ZIP_QUE_PRI));
 232
 233         zip_msg("QUE_PRI %016llx", zip_reg_read(zip->reg_base + ZIP_QUE_PRI));
 234
 235         return 0;
 236 }
 237
 238 static int zip_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 239 {
 240         struct device *dev = &pdev->dev;
 241         struct zip_device *zip = NULL;
 242         int    err;
 243
 244         zip = zip_alloc_device(pdev);
 245         if (!zip)
 246                 return -ENOMEM;
 247
 248         dev_info(dev, "Found ZIP device %d %x:%x on Node %d\n", zip->index,
 249                  pdev->vendor, pdev->device, dev_to_node(dev));
 250
 251         pci_set_drvdata(pdev, zip);
 252         zip->pdev = pdev;
 253
 254         err = pci_enable_device(pdev);
 255         if (err) {
 256                 dev_err(dev, "Failed to enable PCI device");
 257                 goto err_free_device;
 258         }
 259
 260         err = pci_request_regions(pdev, DRV_NAME);
 261         if (err) {
 262                 dev_err(dev, "PCI request regions failed 0x%x", err);
 263                 goto err_disable_device;
 264         }
 265
 266         err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
 267         if (err) {
 268                 dev_err(dev, "Unable to get usable DMA configuration\n");
 269                 goto err_release_regions;
 270         }
 271
 272         err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
 273         if (err) {
 274                 dev_err(dev, "Unable to get 48-bit DMA for allocations\n");
 275                 goto err_release_regions;
 276         }
 277
 278         /* MAP configuration registers */
 279         zip->reg_base = pci_ioremap_bar(pdev, PCI_CFG_ZIP_PF_BAR0);
 280         if (!zip->reg_base) {
 281                 dev_err(dev, "ZIP: Cannot map BAR0 CSR memory space, aborting");
 282                 err = -ENOMEM;
 283                 goto err_release_regions;
 284         }
 285
 286         /* Initialize ZIP Hardware */
 287         err = zip_init_hw(zip);
 288         if (err)
 289                 goto err_release_regions;
 290
 291         return 0;
 292
 293 err_release_regions:
 294         if (zip->reg_base)
 295                 iounmap(zip->reg_base);
 296         pci_release_regions(pdev);
 297
 298 err_disable_device:
 299         pci_disable_device(pdev);
 300
 301 err_free_device:
 302         pci_set_drvdata(pdev, NULL);
 303
 304         /* Remove zip_dev from zip_device list, free the zip_device memory */
 305         zip_dev[zip->index] = NULL;
 306         devm_kfree(dev, zip);
 307
 308         return err;
 309 }
 310
 311 static void zip_remove(struct pci_dev *pdev)
 312 {
 313         struct zip_device *zip = pci_get_drvdata(pdev);
 314         union zip_cmd_ctl cmd_ctl;
 315         int q = 0;
 316
 317         if (!zip)
 318                 return;
 319
 320         if (zip->reg_base) {
 321                 cmd_ctl.u_reg64 = 0x0ull;
 322                 cmd_ctl.s.reset = 1;  /* Forces ZIP cores to do reset */
 323                 zip_reg_write(cmd_ctl.u_reg64, (zip->reg_base + ZIP_CMD_CTL));
 324                 iounmap(zip->reg_base);
 325         }
 326
 327         pci_release_regions(pdev);
 328         pci_disable_device(pdev);
 329
 330         /*
 331          * Free Command Queue buffers. This free should be called for all
 332          * the enabled Queues.
 333          */
 334         for (q = 0; q < ZIP_NUM_QUEUES; q++)
 335                 zip_cmd_qbuf_free(zip, q);
 336
 337         pci_set_drvdata(pdev, NULL);
 338         /* remove zip device from zip device list */
 339         zip_dev[zip->index] = NULL;
 340 }
 341
 342 /* PCI Sub-System Interface */
 343 static struct pci_driver zip_driver = {
 344         .name       =  DRV_NAME,
 345         .id_table   =  zip_id_table,
 346         .probe      =  zip_probe,
 347         .remove     =  zip_remove,
 348 };
 349
 350 /* Kernel Crypto Subsystem Interface */
 351
 352 static struct crypto_alg zip_comp_deflate = {
 353         .cra_name               = "deflate",
 354         .cra_flags              = CRYPTO_ALG_TYPE_COMPRESS,
 355         .cra_ctxsize            = sizeof(struct zip_kernel_ctx),
 356         .cra_priority           = 300,
 357         .cra_module             = THIS_MODULE,
 358         .cra_init               = zip_alloc_comp_ctx_deflate,
 359         .cra_exit               = zip_free_comp_ctx,
 360         .cra_u                  = { .compress = {
 361                 .coa_compress   = zip_comp_compress,
 362                 .coa_decompress = zip_comp_decompress
 363                  } }
 364 };
 365
 366 static struct crypto_alg zip_comp_lzs = {
 367         .cra_name               = "lzs",
 368         .cra_flags              = CRYPTO_ALG_TYPE_COMPRESS,
 369         .cra_ctxsize            = sizeof(struct zip_kernel_ctx),
 370         .cra_priority           = 300,
 371         .cra_module             = THIS_MODULE,
 372         .cra_init               = zip_alloc_comp_ctx_lzs,
 373         .cra_exit               = zip_free_comp_ctx,
 374         .cra_u                  = { .compress = {
 375                 .coa_compress   = zip_comp_compress,
 376                 .coa_decompress = zip_comp_decompress
 377                  } }
 378 };
 379
 380 static struct scomp_alg zip_scomp_deflate = {
 381         .alloc_ctx              = zip_alloc_scomp_ctx_deflate,
 382         .free_ctx               = zip_free_scomp_ctx,
 383         .compress               = zip_scomp_compress,
 384         .decompress             = zip_scomp_decompress,
 385         .base                   = {
 386                 .cra_name               = "deflate",
 387                 .cra_driver_name        = "deflate-scomp",
 388                 .cra_module             = THIS_MODULE,
 389                 .cra_priority           = 300,
 390         }
 391 };
 392
 393 static struct scomp_alg zip_scomp_lzs = {
 394         .alloc_ctx              = zip_alloc_scomp_ctx_lzs,
 395         .free_ctx               = zip_free_scomp_ctx,
 396         .compress               = zip_scomp_compress,
 397         .decompress             = zip_scomp_decompress,
 398         .base                   = {
 399                 .cra_name               = "lzs",
 400                 .cra_driver_name        = "lzs-scomp",
 401                 .cra_module             = THIS_MODULE,
 402                 .cra_priority           = 300,
 403         }
 404 };
 405
 406 static int zip_register_compression_device(void)
 407 {
 408         int ret;
 409
 410         ret = crypto_register_alg(&zip_comp_deflate);
 411         if (ret < 0) {
 412                 zip_err("Deflate algorithm registration failed\n");
 413                 return ret;
 414         }
 415
 416         ret = crypto_register_alg(&zip_comp_lzs);
 417         if (ret < 0) {
 418                 zip_err("LZS algorithm registration failed\n");
 419                 goto err_unregister_alg_deflate;
 420         }
 421
 422         ret = crypto_register_scomp(&zip_scomp_deflate);
 423         if (ret < 0) {
 424                 zip_err("Deflate scomp algorithm registration failed\n");
 425                 goto err_unregister_alg_lzs;
 426         }
 427
 428         ret = crypto_register_scomp(&zip_scomp_lzs);
 429         if (ret < 0) {
 430                 zip_err("LZS scomp algorithm registration failed\n");
 431                 goto err_unregister_scomp_deflate;
 432         }
 433
 434         return ret;
 435
 436 err_unregister_scomp_deflate:
 437         crypto_unregister_scomp(&zip_scomp_deflate);
 438 err_unregister_alg_lzs:
 439         crypto_unregister_alg(&zip_comp_lzs);
 440 err_unregister_alg_deflate:
 441         crypto_unregister_alg(&zip_comp_deflate);
 442
 443         return ret;
 444 }
 445
 446 static void zip_unregister_compression_device(void)
 447 {
 448         crypto_unregister_alg(&zip_comp_deflate);
 449         crypto_unregister_alg(&zip_comp_lzs);
 450         crypto_unregister_scomp(&zip_scomp_deflate);
 451         crypto_unregister_scomp(&zip_scomp_lzs);
 452 }
 453
 454 /*
 455  * debugfs functions
 456  */
 457 #ifdef CONFIG_DEBUG_FS
 458 #include <linux/debugfs.h>
 459
 460 /* Displays ZIP device statistics */
 461 static int zip_show_stats(struct seq_file *s, void *unused)
 462 {
 463         u64 val = 0ull;
 464         u64 avg_chunk = 0ull, avg_cr = 0ull;
 465         u32 q = 0;
 466
 467         int index  = 0;
 468         struct zip_device *zip;
 469         struct zip_stats  *st;
 470
 471         for (index = 0; index < MAX_ZIP_DEVICES; index++) {
 472                 if (zip_dev[index]) {
 473                         zip = zip_dev[index];
 474                         st  = &zip->stats;
 475
 476                         /* Get all the pending requests */
 477                         for (q = 0; q < ZIP_NUM_QUEUES; q++) {
 478                                 val = zip_reg_read((zip->reg_base +
 479                                                     ZIP_DBG_COREX_STA(q)));
 480                                 val = (val >> 32);
 481                                 val = val & 0xffffff;
 482                                 atomic64_add(val, &st->pending_req);
 483                         }
 484
 485                         avg_chunk = (atomic64_read(&st->comp_in_bytes) /
 486                                      atomic64_read(&st->comp_req_complete));
 487                         avg_cr = (atomic64_read(&st->comp_in_bytes) /
 488                                   atomic64_read(&st->comp_out_bytes));
 489                         seq_printf(s, "        ZIP Device %d Stats\n"
 490                                       "-----------------------------------\n"
 491                                       "Comp Req Submitted        : \t%lld\n"
 492                                       "Comp Req Completed        : \t%lld\n"
 493                                       "Compress In Bytes         : \t%lld\n"
 494                                       "Compressed Out Bytes      : \t%lld\n"
 495                                       "Average Chunk size        : \t%llu\n"
 496                                       "Average Compression ratio : \t%llu\n"
 497                                       "Decomp Req Submitted      : \t%lld\n"
 498                                       "Decomp Req Completed      : \t%lld\n"
 499                                       "Decompress In Bytes       : \t%lld\n"
 500                                       "Decompressed Out Bytes    : \t%lld\n"
 501                                       "Decompress Bad requests   : \t%lld\n"
 502                                       "Pending Req               : \t%lld\n"
 503                                         "---------------------------------\n",
 504                                        index,
 505                                        (u64)atomic64_read(&st->comp_req_submit),
 506                                        (u64)atomic64_read(&st->comp_req_complete),
 507                                        (u64)atomic64_read(&st->comp_in_bytes),
 508                                        (u64)atomic64_read(&st->comp_out_bytes),
 509                                        avg_chunk,
 510                                        avg_cr,
 511                                        (u64)atomic64_read(&st->decomp_req_submit),
 512                                        (u64)atomic64_read(&st->decomp_req_complete),
 513                                        (u64)atomic64_read(&st->decomp_in_bytes),
 514                                        (u64)atomic64_read(&st->decomp_out_bytes),
 515                                        (u64)atomic64_read(&st->decomp_bad_reqs),
 516                                        (u64)atomic64_read(&st->pending_req));
 517
 518                         /* Reset pending requests  count */
 519                         atomic64_set(&st->pending_req, 0);
 520                 }
 521         }
 522         return 0;
 523 }
 524
 525 /* Clears stats data */
 526 static int zip_clear_stats(struct seq_file *s, void *unused)
 527 {
 528         int index = 0;
 529
 530         for (index = 0; index < MAX_ZIP_DEVICES; index++) {
 531                 if (zip_dev[index]) {
 532                         memset(&zip_dev[index]->stats, 0,
 533                                sizeof(struct zip_stats));
 534                         seq_printf(s, "Cleared stats for zip %d\n", index);
 535                 }
 536         }
 537
 538         return 0;
 539 }
 540
 541 static struct zip_registers zipregs[64] = {
 542         {"ZIP_CMD_CTL        ",  0x0000ull},
 543         {"ZIP_THROTTLE       ",  0x0010ull},
 544         {"ZIP_CONSTANTS      ",  0x00A0ull},
 545         {"ZIP_QUE0_MAP       ",  0x1400ull},
 546         {"ZIP_QUE1_MAP       ",  0x1408ull},
 547         {"ZIP_QUE_ENA        ",  0x0500ull},
 548         {"ZIP_QUE_PRI        ",  0x0508ull},
 549         {"ZIP_QUE0_DONE      ",  0x2000ull},
 550         {"ZIP_QUE1_DONE      ",  0x2008ull},
 551         {"ZIP_QUE0_DOORBELL  ",  0x4000ull},
 552         {"ZIP_QUE1_DOORBELL  ",  0x4008ull},
 553         {"ZIP_QUE0_SBUF_ADDR ",  0x1000ull},
 554         {"ZIP_QUE1_SBUF_ADDR ",  0x1008ull},
 555         {"ZIP_QUE0_SBUF_CTL  ",  0x1200ull},
 556         {"ZIP_QUE1_SBUF_CTL  ",  0x1208ull},
 557         { NULL, 0}
 558 };
 559
 560 /* Prints registers' contents */
 561 static int zip_print_regs(struct seq_file *s, void *unused)
 562 {
 563         u64 val = 0;
 564         int i = 0, index = 0;
 565
 566         for (index = 0; index < MAX_ZIP_DEVICES; index++) {
 567                 if (zip_dev[index]) {
 568                         seq_printf(s, "--------------------------------\n"
 569                                       "     ZIP Device %d Registers\n"
 570                                       "--------------------------------\n",
 571                                       index);
 572
 573                         i = 0;
 574
 575                         while (zipregs[i].reg_name) {
 576                                 val = zip_reg_read((zip_dev[index]->reg_base +
 577                                                     zipregs[i].reg_offset));
 578                                 seq_printf(s, "%s: 0x%016llx\n",
 579                                            zipregs[i].reg_name, val);
 580                                 i++;
 581                         }
 582                 }
 583         }
 584         return 0;
 585 }
 586
 587 static int zip_stats_open(struct inode *inode, struct file *file)
 588 {
 589         return single_open(file, zip_show_stats, NULL);
 590 }
 591
 592 static const struct file_operations zip_stats_fops = {
 593         .owner = THIS_MODULE,
 594         .open  = zip_stats_open,
 595         .read  = seq_read,
 596 };
 597
 598 static int zip_clear_open(struct inode *inode, struct file *file)
 599 {
 600         return single_open(file, zip_clear_stats, NULL);
 601 }
 602
 603 static const struct file_operations zip_clear_fops = {
 604         .owner = THIS_MODULE,
 605         .open  = zip_clear_open,
 606         .read  = seq_read,
 607 };
 608
 609 static int zip_regs_open(struct inode *inode, struct file *file)
 610 {
 611         return single_open(file, zip_print_regs, NULL);
 612 }
 613
 614 static const struct file_operations zip_regs_fops = {
 615         .owner = THIS_MODULE,
 616         .open  = zip_regs_open,
 617         .read  = seq_read,
 618 };
 619
 620 /* Root directory for thunderx_zip debugfs entry */
 621 static struct dentry *zip_debugfs_root;
 622
 623 static int __init zip_debugfs_init(void)
 624 {
 625         struct dentry *zip_stats, *zip_clear, *zip_regs;
 626
 627         if (!debugfs_initialized())
 628                 return -ENODEV;
 629
 630         zip_debugfs_root = debugfs_create_dir("thunderx_zip", NULL);
 631         if (!zip_debugfs_root)
 632                 return -ENOMEM;
 633
 634         /* Creating files for entries inside thunderx_zip directory */
 635         zip_stats = debugfs_create_file("zip_stats", 0444,
 636                                         zip_debugfs_root,
 637                                         NULL, &zip_stats_fops);
 638         if (!zip_stats)
 639                 goto failed_to_create;
 640
 641         zip_clear = debugfs_create_file("zip_clear", 0444,
 642                                         zip_debugfs_root,
 643                                         NULL, &zip_clear_fops);
 644         if (!zip_clear)
 645                 goto failed_to_create;
 646
 647         zip_regs = debugfs_create_file("zip_regs", 0444,
 648                                        zip_debugfs_root,
 649                                        NULL, &zip_regs_fops);
 650         if (!zip_regs)
 651                 goto failed_to_create;
 652
 653         return 0;
 654
 655 failed_to_create:
 656         debugfs_remove_recursive(zip_debugfs_root);
 657         return -ENOENT;
 658 }
 659
 660 static void __exit zip_debugfs_exit(void)
 661 {
 662         debugfs_remove_recursive(zip_debugfs_root);
 663 }
 664
 665 #else
 666 static int __init zip_debugfs_init(void)
 667 {
 668         return 0;
 669 }
 670
 671 static void __exit zip_debugfs_exit(void) { }
 672
 673 #endif
 674 /* debugfs - end */
 675
 676 static int __init zip_init_module(void)
 677 {
 678         int ret;
 679
 680         zip_msg("%s\n", DRV_NAME);
 681
 682         ret = pci_register_driver(&zip_driver);
 683         if (ret < 0) {
 684                 zip_err("ZIP: pci_register_driver() failed\n");
 685                 return ret;
 686         }
 687
 688         /* Register with the Kernel Crypto Interface */
 689         ret = zip_register_compression_device();
 690         if (ret < 0) {
 691                 zip_err("ZIP: Kernel Crypto Registration failed\n");
 692                 goto err_pci_unregister;
 693         }
 694
 695         /* comp-decomp statistics are handled with debugfs interface */
 696         ret = zip_debugfs_init();
 697         if (ret < 0) {
 698                 zip_err("ZIP: debugfs initialization failed\n");
 699                 goto err_crypto_unregister;
 700         }
 701
 702         return ret;
 703
 704 err_crypto_unregister:
 705         zip_unregister_compression_device();
 706
 707 err_pci_unregister:
 708         pci_unregister_driver(&zip_driver);
 709         return ret;
 710 }
 711
 712 static void __exit zip_cleanup_module(void)
 713 {
 714         zip_debugfs_exit();
 715
 716         /* Unregister from the kernel crypto interface */
 717         zip_unregister_compression_device();
 718
 719         /* Unregister this driver for pci zip devices */
 720         pci_unregister_driver(&zip_driver);
 721 }
 722
 723 module_init(zip_init_module);
 724 module_exit(zip_cleanup_module);
 725
 726 MODULE_AUTHOR("Cavium Inc");
 727 MODULE_DESCRIPTION("Cavium Inc ThunderX ZIP Driver");
 728 MODULE_LICENSE("GPL v2");
 729 MODULE_DEVICE_TABLE(pci, zip_id_table);