drivers/misc/genwqe/card_ddcb.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /**
   3  * IBM Accelerator Family 'GenWQE'
   4  *
   5  * (C) Copyright IBM Corp. 2013
   6  *
   7  * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
   8  * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
   9  * Author: Michael Jung <mijung@gmx.net>
  10  * Author: Michael Ruettger <michael@ibmra.de>
  11  */
  12
  13 /*
  14  * Device Driver Control Block (DDCB) queue support. Definition of
  15  * interrupt handlers for queue support as well as triggering the
  16  * health monitor code in case of problems. The current hardware uses
  17  * an MSI interrupt which is shared between error handling and
  18  * functional code.
  19  */
  20
  21 #include <linux/types.h>
  22 #include <linux/sched.h>
  23 #include <linux/wait.h>
  24 #include <linux/pci.h>
  25 #include <linux/string.h>
  26 #include <linux/dma-mapping.h>
  27 #include <linux/delay.h>
  28 #include <linux/module.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/crc-itu-t.h>
  31
  32 #include "card_base.h"
  33 #include "card_ddcb.h"
  34
  35 /*
  36  * N: next DDCB, this is where the next DDCB will be put.
  37  * A: active DDCB, this is where the code will look for the next completion.
  38  * x: DDCB is enqueued, we are waiting for its completion.
  39
  40  * Situation (1): Empty queue
  41  *  +---+---+---+---+---+---+---+---+
  42  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  43  *  |   |   |   |   |   |   |   |   |
  44  *  +---+---+---+---+---+---+---+---+
  45  *           A/N
  46  *  enqueued_ddcbs = A - N = 2 - 2 = 0
  47  *
  48  * Situation (2): Wrapped, N > A
  49  *  +---+---+---+---+---+---+---+---+
  50  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  51  *  |   |   | x | x |   |   |   |   |
  52  *  +---+---+---+---+---+---+---+---+
  53  *            A       N
  54  *  enqueued_ddcbs = N - A = 4 - 2 = 2
  55  *
  56  * Situation (3): Queue wrapped, A > N
  57  *  +---+---+---+---+---+---+---+---+
  58  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  59  *  | x | x |   |   | x | x | x | x |
  60  *  +---+---+---+---+---+---+---+---+
  61  *            N       A
  62  *  enqueued_ddcbs = queue_max  - (A - N) = 8 - (4 - 2) = 6
  63  *
  64  * Situation (4a): Queue full N > A
  65  *  +---+---+---+---+---+---+---+---+
  66  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  67  *  | x | x | x | x | x | x | x |   |
  68  *  +---+---+---+---+---+---+---+---+
  69  *    A                           N
  70  *
  71  *  enqueued_ddcbs = N - A = 7 - 0 = 7
  72  *
  73  * Situation (4a): Queue full A > N
  74  *  +---+---+---+---+---+---+---+---+
  75  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  76  *  | x | x | x |   | x | x | x | x |
  77  *  +---+---+---+---+---+---+---+---+
  78  *                N   A
  79  *  enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7
  80  */
  81
  82 static int queue_empty(struct ddcb_queue *queue)
  83 {
  84         return queue->ddcb_next == queue->ddcb_act;
  85 }
  86
  87 static int queue_enqueued_ddcbs(struct ddcb_queue *queue)
  88 {
  89         if (queue->ddcb_next >= queue->ddcb_act)
  90                 return queue->ddcb_next - queue->ddcb_act;
  91
  92         return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next);
  93 }
  94
  95 static int queue_free_ddcbs(struct ddcb_queue *queue)
  96 {
  97         int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1;
  98
  99         if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */
 100                 return 0;
 101         }
 102         return free_ddcbs;
 103 }
 104
 105 /*
 106  * Use of the PRIV field in the DDCB for queue debugging:
 107  *
 108  * (1) Trying to get rid of a DDCB which saw a timeout:
 109  *     pddcb->priv[6] = 0xcc;   # cleared
 110  *
 111  * (2) Append a DDCB via NEXT bit:
 112  *     pddcb->priv[7] = 0xaa;   # appended
 113  *
 114  * (3) DDCB needed tapping:
 115  *     pddcb->priv[7] = 0xbb;   # tapped
 116  *
 117  * (4) DDCB marked as correctly finished:
 118  *     pddcb->priv[6] = 0xff;   # finished
 119  */
 120
 121 static inline void ddcb_mark_tapped(struct ddcb *pddcb)
 122 {
 123         pddcb->priv[7] = 0xbb;  /* tapped */
 124 }
 125
 126 static inline void ddcb_mark_appended(struct ddcb *pddcb)
 127 {
 128         pddcb->priv[7] = 0xaa;  /* appended */
 129 }
 130
 131 static inline void ddcb_mark_cleared(struct ddcb *pddcb)
 132 {
 133         pddcb->priv[6] = 0xcc; /* cleared */
 134 }
 135
 136 static inline void ddcb_mark_finished(struct ddcb *pddcb)
 137 {
 138         pddcb->priv[6] = 0xff;  /* finished */
 139 }
 140
 141 static inline void ddcb_mark_unused(struct ddcb *pddcb)
 142 {
 143         pddcb->priv_64 = cpu_to_be64(0); /* not tapped */
 144 }
 145
 146 /**
 147  * genwqe_crc16() - Generate 16-bit crc as required for DDCBs
 148  * @buff:       pointer to data buffer
 149  * @len:        length of data for calculation
 150  * @init:       initial crc (0xffff at start)
 151  *
 152  * Polynomial = x^16 + x^12 + x^5 + 1   (0x1021)
 153  * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff
 154  *          should result in a crc16 of 0x89c3
 155  *
 156  * Return: crc16 checksum in big endian format !
 157  */
 158 static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init)
 159 {
 160         return crc_itu_t(init, buff, len);
 161 }
 162
 163 static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue)
 164 {
 165         int i;
 166         struct ddcb *pddcb;
 167         unsigned long flags;
 168         struct pci_dev *pci_dev = cd->pci_dev;
 169
 170         spin_lock_irqsave(&cd->print_lock, flags);
 171
 172         dev_info(&pci_dev->dev,
 173                  "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n",
 174                  cd->card_idx, queue->ddcb_act, queue->ddcb_next);
 175
 176         pddcb = queue->ddcb_vaddr;
 177         for (i = 0; i < queue->ddcb_max; i++) {
 178                 dev_err(&pci_dev->dev,
 179                         "  %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n",
 180                         i == queue->ddcb_act ? '>' : ' ',
 181                         i,
 182                         be16_to_cpu(pddcb->retc_16),
 183                         be16_to_cpu(pddcb->seqnum_16),
 184                         pddcb->hsi,
 185                         pddcb->shi,
 186                         be64_to_cpu(pddcb->priv_64),
 187                         pddcb->cmd);
 188                 pddcb++;
 189         }
 190         spin_unlock_irqrestore(&cd->print_lock, flags);
 191 }
 192
 193 struct genwqe_ddcb_cmd *ddcb_requ_alloc(void)
 194 {
 195         struct ddcb_requ *req;
 196
 197         req = kzalloc(sizeof(*req), GFP_KERNEL);
 198         if (!req)
 199                 return NULL;
 200
 201         return &req->cmd;
 202 }
 203
 204 void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd)
 205 {
 206         struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
 207
 208         kfree(req);
 209 }
 210
 211 static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req)
 212 {
 213         return req->req_state;
 214 }
 215
 216 static inline void ddcb_requ_set_state(struct ddcb_requ *req,
 217                                        enum genwqe_requ_state new_state)
 218 {
 219         req->req_state = new_state;
 220 }
 221
 222 static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req)
 223 {
 224         return req->cmd.ddata_addr != 0x0;
 225 }
 226
 227 /**
 228  * ddcb_requ_finished() - Returns the hardware state of the associated DDCB
 229  * @cd:          pointer to genwqe device descriptor
 230  * @req:         DDCB work request
 231  *
 232  * Status of ddcb_requ mirrors this hardware state, but is copied in
 233  * the ddcb_requ on interrupt/polling function. The lowlevel code
 234  * should check the hardware state directly, the higher level code
 235  * should check the copy.
 236  *
 237  * This function will also return true if the state of the queue is
 238  * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the
 239  * shutdown case.
 240  */
 241 static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req)
 242 {
 243         return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) ||
 244                 (cd->card_state != GENWQE_CARD_USED);
 245 }
 246
 247 /**
 248  * enqueue_ddcb() - Enqueue a DDCB
 249  * @cd:         pointer to genwqe device descriptor
 250  * @queue:      queue this operation should be done on
 251  * @ddcb_no:    pointer to ddcb number being tapped
 252  *
 253  * Start execution of DDCB by tapping or append to queue via NEXT
 254  * bit. This is done by an atomic 'compare and swap' instruction and
 255  * checking SHI and HSI of the previous DDCB.
 256  *
 257  * This function must only be called with ddcb_lock held.
 258  *
 259  * Return: 1 if new DDCB is appended to previous
 260  *         2 if DDCB queue is tapped via register/simulation
 261  */
 262 #define RET_DDCB_APPENDED 1
 263 #define RET_DDCB_TAPPED   2
 264
 265 static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue,
 266                         struct ddcb *pddcb, int ddcb_no)
 267 {
 268         unsigned int try;
 269         int prev_no;
 270         struct ddcb *prev_ddcb;
 271         __be32 old, new, icrc_hsi_shi;
 272         u64 num;
 273
 274         /*
 275          * For performance checks a Dispatch Timestamp can be put into
 276          * DDCB It is supposed to use the SLU's free running counter,
 277          * but this requires PCIe cycles.
 278          */
 279         ddcb_mark_unused(pddcb);
 280
 281         /* check previous DDCB if already fetched */
 282         prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1;
 283         prev_ddcb = &queue->ddcb_vaddr[prev_no];
 284
 285         /*
 286          * It might have happened that the HSI.FETCHED bit is
 287          * set. Retry in this case. Therefore I expect maximum 2 times
 288          * trying.
 289          */
 290         ddcb_mark_appended(pddcb);
 291         for (try = 0; try < 2; try++) {
 292                 old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
 293
 294                 /* try to append via NEXT bit if prev DDCB is not completed */
 295                 if ((old & DDCB_COMPLETED_BE32) != 0x00000000)
 296                         break;
 297
 298                 new = (old | DDCB_NEXT_BE32);
 299
 300                 wmb();          /* need to ensure write ordering */
 301                 icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new);
 302
 303                 if (icrc_hsi_shi == old)
 304                         return RET_DDCB_APPENDED; /* appended to queue */
 305         }
 306
 307         /* Queue must be re-started by updating QUEUE_OFFSET */
 308         ddcb_mark_tapped(pddcb);
 309         num = (u64)ddcb_no << 8;
 310
 311         wmb();                  /* need to ensure write ordering */
 312         __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */
 313
 314         return RET_DDCB_TAPPED;
 315 }
 316
 317 /**
 318  * copy_ddcb_results() - Copy output state from real DDCB to request
 319  *
 320  * Copy DDCB ASV to request struct. There is no endian
 321  * conversion made, since data structure in ASV is still
 322  * unknown here.
 323  *
 324  * This is needed by:
 325  *   - genwqe_purge_ddcb()
 326  *   - genwqe_check_ddcb_queue()
 327  */
 328 static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no)
 329 {
 330         struct ddcb_queue *queue = req->queue;
 331         struct ddcb *pddcb = &queue->ddcb_vaddr[req->num];
 332
 333         memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH);
 334
 335         /* copy status flags of the variant part */
 336         req->cmd.vcrc     = be16_to_cpu(pddcb->vcrc_16);
 337         req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64);
 338         req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64);
 339
 340         req->cmd.attn     = be16_to_cpu(pddcb->attn_16);
 341         req->cmd.progress = be32_to_cpu(pddcb->progress_32);
 342         req->cmd.retc     = be16_to_cpu(pddcb->retc_16);
 343
 344         if (ddcb_requ_collect_debug_data(req)) {
 345                 int prev_no = (ddcb_no == 0) ?
 346                         queue->ddcb_max - 1 : ddcb_no - 1;
 347                 struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no];
 348
 349                 memcpy(&req->debug_data.ddcb_finished, pddcb,
 350                        sizeof(req->debug_data.ddcb_finished));
 351                 memcpy(&req->debug_data.ddcb_prev, prev_pddcb,
 352                        sizeof(req->debug_data.ddcb_prev));
 353         }
 354 }
 355
 356 /**
 357  * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work equests.
 358  * @cd:         pointer to genwqe device descriptor
 359  *
 360  * Return: Number of DDCBs which were finished
 361  */
 362 static int genwqe_check_ddcb_queue(struct genwqe_dev *cd,
 363                                    struct ddcb_queue *queue)
 364 {
 365         unsigned long flags;
 366         int ddcbs_finished = 0;
 367         struct pci_dev *pci_dev = cd->pci_dev;
 368
 369         spin_lock_irqsave(&queue->ddcb_lock, flags);
 370
 371         /* FIXME avoid soft locking CPU */
 372         while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) {
 373
 374                 struct ddcb *pddcb;
 375                 struct ddcb_requ *req;
 376                 u16 vcrc, vcrc_16, retc_16;
 377
 378                 pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
 379
 380                 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) ==
 381                     0x00000000)
 382                         goto go_home; /* not completed, continue waiting */
 383
 384                 wmb();  /*  Add sync to decouple prev. read operations */
 385
 386                 /* Note: DDCB could be purged */
 387                 req = queue->ddcb_req[queue->ddcb_act];
 388                 if (req == NULL) {
 389                         /* this occurs if DDCB is purged, not an error */
 390                         /* Move active DDCB further; Nothing to do anymore. */
 391                         goto pick_next_one;
 392                 }
 393
 394                 /*
 395                  * HSI=0x44 (fetched and completed), but RETC is
 396                  * 0x101, or even worse 0x000.
 397                  *
 398                  * In case of seeing the queue in inconsistent state
 399                  * we read the errcnts and the queue status to provide
 400                  * a trigger for our PCIe analyzer stop capturing.
 401                  */
 402                 retc_16 = be16_to_cpu(pddcb->retc_16);
 403                 if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) {
 404                         u64 errcnts, status;
 405                         u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr;
 406
 407                         errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS);
 408                         status  = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
 409
 410                         dev_err(&pci_dev->dev,
 411                                 "[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n",
 412                                 __func__, be16_to_cpu(pddcb->seqnum_16),
 413                                 pddcb->hsi, retc_16, errcnts, status,
 414                                 queue->ddcb_daddr + ddcb_offs);
 415                 }
 416
 417                 copy_ddcb_results(req, queue->ddcb_act);
 418                 queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */
 419
 420                 dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num);
 421                 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 422
 423                 ddcb_mark_finished(pddcb);
 424
 425                 /* calculate CRC_16 to see if VCRC is correct */
 426                 vcrc = genwqe_crc16(pddcb->asv,
 427                                    VCRC_LENGTH(req->cmd.asv_length),
 428                                    0xffff);
 429                 vcrc_16 = be16_to_cpu(pddcb->vcrc_16);
 430                 if (vcrc != vcrc_16) {
 431                         printk_ratelimited(KERN_ERR
 432                                 "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n",
 433                                 GENWQE_DEVNAME, dev_name(&pci_dev->dev),
 434                                 pddcb->pre, VCRC_LENGTH(req->cmd.asv_length),
 435                                 vcrc, vcrc_16);
 436                 }
 437
 438                 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
 439                 queue->ddcbs_completed++;
 440                 queue->ddcbs_in_flight--;
 441
 442                 /* wake up process waiting for this DDCB, and
 443                    processes on the busy queue */
 444                 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
 445                 wake_up_interruptible(&queue->busy_waitq);
 446
 447 pick_next_one:
 448                 queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max;
 449                 ddcbs_finished++;
 450         }
 451
 452  go_home:
 453         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 454         return ddcbs_finished;
 455 }
 456
 457 /**
 458  * __genwqe_wait_ddcb(): Waits until DDCB is completed
 459  * @cd:         pointer to genwqe device descriptor
 460  * @req:        pointer to requsted DDCB parameters
 461  *
 462  * The Service Layer will update the RETC in DDCB when processing is
 463  * pending or done.
 464  *
 465  * Return: > 0 remaining jiffies, DDCB completed
 466  *           -ETIMEDOUT when timeout
 467  *           -ERESTARTSYS when ^C
 468  *           -EINVAL when unknown error condition
 469  *
 470  * When an error is returned the called needs to ensure that
 471  * purge_ddcb() is being called to get the &req removed from the
 472  * queue.
 473  */
 474 int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
 475 {
 476         int rc;
 477         unsigned int ddcb_no;
 478         struct ddcb_queue *queue;
 479         struct pci_dev *pci_dev = cd->pci_dev;
 480
 481         if (req == NULL)
 482                 return -EINVAL;
 483
 484         queue = req->queue;
 485         if (queue == NULL)
 486                 return -EINVAL;
 487
 488         ddcb_no = req->num;
 489         if (ddcb_no >= queue->ddcb_max)
 490                 return -EINVAL;
 491
 492         rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no],
 493                                 ddcb_requ_finished(cd, req),
 494                                 GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ);
 495
 496         /*
 497          * We need to distinguish 3 cases here:
 498          *   1. rc == 0              timeout occured
 499          *   2. rc == -ERESTARTSYS   signal received
 500          *   3. rc > 0               remaining jiffies condition is true
 501          */
 502         if (rc == 0) {
 503                 struct ddcb_queue *queue = req->queue;
 504                 struct ddcb *pddcb;
 505
 506                 /*
 507                  * Timeout may be caused by long task switching time.
 508                  * When timeout happens, check if the request has
 509                  * meanwhile completed.
 510                  */
 511                 genwqe_check_ddcb_queue(cd, req->queue);
 512                 if (ddcb_requ_finished(cd, req))
 513                         return rc;
 514
 515                 dev_err(&pci_dev->dev,
 516                         "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n",
 517                         __func__, req->num, rc, ddcb_requ_get_state(req),
 518                         req);
 519                 dev_err(&pci_dev->dev,
 520                         "[%s]      IO_QUEUE_STATUS=0x%016llx\n", __func__,
 521                         __genwqe_readq(cd, queue->IO_QUEUE_STATUS));
 522
 523                 pddcb = &queue->ddcb_vaddr[req->num];
 524                 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 525
 526                 print_ddcb_info(cd, req->queue);
 527                 return -ETIMEDOUT;
 528
 529         } else if (rc == -ERESTARTSYS) {
 530                 return rc;
 531                 /*
 532                  * EINTR:       Stops the application
 533                  * ERESTARTSYS: Restartable systemcall; called again
 534                  */
 535
 536         } else if (rc < 0) {
 537                 dev_err(&pci_dev->dev,
 538                         "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n",
 539                         __func__, req->num, rc, ddcb_requ_get_state(req));
 540                 return -EINVAL;
 541         }
 542
 543         /* Severe error occured. Driver is forced to stop operation */
 544         if (cd->card_state != GENWQE_CARD_USED) {
 545                 dev_err(&pci_dev->dev,
 546                         "[%s] err: DDCB#%d forced to stop (rc=%d)\n",
 547                         __func__, req->num, rc);
 548                 return -EIO;
 549         }
 550         return rc;
 551 }
 552
 553 /**
 554  * get_next_ddcb() - Get next available DDCB
 555  * @cd:         pointer to genwqe device descriptor
 556  *
 557  * DDCB's content is completely cleared but presets for PRE and
 558  * SEQNUM. This function must only be called when ddcb_lock is held.
 559  *
 560  * Return: NULL if no empty DDCB available otherwise ptr to next DDCB.
 561  */
 562 static struct ddcb *get_next_ddcb(struct genwqe_dev *cd,
 563                                   struct ddcb_queue *queue,
 564                                   int *num)
 565 {
 566         u64 *pu64;
 567         struct ddcb *pddcb;
 568
 569         if (queue_free_ddcbs(queue) == 0) /* queue is  full */
 570                 return NULL;
 571
 572         /* find new ddcb */
 573         pddcb = &queue->ddcb_vaddr[queue->ddcb_next];
 574
 575         /* if it is not completed, we are not allowed to use it */
 576         /* barrier(); */
 577         if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000)
 578                 return NULL;
 579
 580         *num = queue->ddcb_next;        /* internal DDCB number */
 581         queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max;
 582
 583         /* clear important DDCB fields */
 584         pu64 = (u64 *)pddcb;
 585         pu64[0] = 0ULL;         /* offs 0x00 (ICRC,HSI,SHI,...) */
 586         pu64[1] = 0ULL;         /* offs 0x01 (ACFUNC,CMD...) */
 587
 588         /* destroy previous results in ASV */
 589         pu64[0x80/8] = 0ULL;    /* offs 0x80 (ASV + 0) */
 590         pu64[0x88/8] = 0ULL;    /* offs 0x88 (ASV + 0x08) */
 591         pu64[0x90/8] = 0ULL;    /* offs 0x90 (ASV + 0x10) */
 592         pu64[0x98/8] = 0ULL;    /* offs 0x98 (ASV + 0x18) */
 593         pu64[0xd0/8] = 0ULL;    /* offs 0xd0 (RETC,ATTN...) */
 594
 595         pddcb->pre = DDCB_PRESET_PRE; /* 128 */
 596         pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++);
 597         return pddcb;
 598 }
 599
 600 /**
 601  * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue
 602  * @cd:         genwqe device descriptor
 603  * @req:        DDCB request
 604  *
 605  * This will fail when the request was already FETCHED. In this case
 606  * we need to wait until it is finished. Else the DDCB can be
 607  * reused. This function also ensures that the request data structure
 608  * is removed from ddcb_req[].
 609  *
 610  * Do not forget to call this function when genwqe_wait_ddcb() fails,
 611  * such that the request gets really removed from ddcb_req[].
 612  *
 613  * Return: 0 success
 614  */
 615 int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
 616 {
 617         struct ddcb *pddcb = NULL;
 618         unsigned int t;
 619         unsigned long flags;
 620         struct ddcb_queue *queue = req->queue;
 621         struct pci_dev *pci_dev = cd->pci_dev;
 622         u64 queue_status;
 623         __be32 icrc_hsi_shi = 0x0000;
 624         __be32 old, new;
 625
 626         /* unsigned long flags; */
 627         if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) {
 628                 dev_err(&pci_dev->dev,
 629                         "[%s] err: software timeout is not set!\n", __func__);
 630                 return -EFAULT;
 631         }
 632
 633         pddcb = &queue->ddcb_vaddr[req->num];
 634
 635         for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) {
 636
 637                 spin_lock_irqsave(&queue->ddcb_lock, flags);
 638
 639                 /* Check if req was meanwhile finished */
 640                 if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED)
 641                         goto go_home;
 642
 643                 /* try to set PURGE bit if FETCHED/COMPLETED are not set */
 644                 old = pddcb->icrc_hsi_shi_32;   /* read SHI/HSI in BE32 */
 645                 if ((old & DDCB_FETCHED_BE32) == 0x00000000) {
 646
 647                         new = (old | DDCB_PURGE_BE32);
 648                         icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32,
 649                                                old, new);
 650                         if (icrc_hsi_shi == old)
 651                                 goto finish_ddcb;
 652                 }
 653
 654                 /* normal finish with HSI bit */
 655                 barrier();
 656                 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
 657                 if (icrc_hsi_shi & DDCB_COMPLETED_BE32)
 658                         goto finish_ddcb;
 659
 660                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 661
 662                 /*
 663                  * Here the check_ddcb() function will most likely
 664                  * discover this DDCB to be finished some point in
 665                  * time. It will mark the req finished and free it up
 666                  * in the list.
 667                  */
 668
 669                 copy_ddcb_results(req, req->num); /* for the failing case */
 670                 msleep(100); /* sleep for 1/10 second and try again */
 671                 continue;
 672
 673 finish_ddcb:
 674                 copy_ddcb_results(req, req->num);
 675                 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
 676                 queue->ddcbs_in_flight--;
 677                 queue->ddcb_req[req->num] = NULL; /* delete from array */
 678                 ddcb_mark_cleared(pddcb);
 679
 680                 /* Move active DDCB further; Nothing to do here anymore. */
 681
 682                 /*
 683                  * We need to ensure that there is at least one free
 684                  * DDCB in the queue. To do that, we must update
 685                  * ddcb_act only if the COMPLETED bit is set for the
 686                  * DDCB we are working on else we treat that DDCB even
 687                  * if we PURGED it as occupied (hardware is supposed
 688                  * to set the COMPLETED bit yet!).
 689                  */
 690                 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
 691                 if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) &&
 692                     (queue->ddcb_act == req->num)) {
 693                         queue->ddcb_act = ((queue->ddcb_act + 1) %
 694                                            queue->ddcb_max);
 695                 }
 696 go_home:
 697                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 698                 return 0;
 699         }
 700
 701         /*
 702          * If the card is dead and the queue is forced to stop, we
 703          * might see this in the queue status register.
 704          */
 705         queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
 706
 707         dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num);
 708         genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 709
 710         dev_err(&pci_dev->dev,
 711                 "[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n",
 712                 __func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT,
 713                 queue_status);
 714
 715         print_ddcb_info(cd, req->queue);
 716
 717         return -EFAULT;
 718 }
 719
 720 int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d)
 721 {
 722         int len;
 723         struct pci_dev *pci_dev = cd->pci_dev;
 724
 725         if (d == NULL) {
 726                 dev_err(&pci_dev->dev,
 727                         "[%s] err: invalid memory for debug data!\n",
 728                         __func__);
 729                 return -EFAULT;
 730         }
 731
 732         len  = sizeof(d->driver_version);
 733         snprintf(d->driver_version, len, "%s", DRV_VERSION);
 734         d->slu_unitcfg = cd->slu_unitcfg;
 735         d->app_unitcfg = cd->app_unitcfg;
 736         return 0;
 737 }
 738
 739 /**
 740  * __genwqe_enqueue_ddcb() - Enqueue a DDCB
 741  * @cd:         pointer to genwqe device descriptor
 742  * @req:        pointer to DDCB execution request
 743  * @f_flags:    file mode: blocking, non-blocking
 744  *
 745  * Return: 0 if enqueuing succeeded
 746  *         -EIO if card is unusable/PCIe problems
 747  *         -EBUSY if enqueuing failed
 748  */
 749 int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,
 750                           unsigned int f_flags)
 751 {
 752         struct ddcb *pddcb;
 753         unsigned long flags;
 754         struct ddcb_queue *queue;
 755         struct pci_dev *pci_dev = cd->pci_dev;
 756         u16 icrc;
 757
 758  retry:
 759         if (cd->card_state != GENWQE_CARD_USED) {
 760                 printk_ratelimited(KERN_ERR
 761                         "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n",
 762                         GENWQE_DEVNAME, dev_name(&pci_dev->dev),
 763                         __func__, req->num);
 764                 return -EIO;
 765         }
 766
 767         queue = req->queue = &cd->queue;
 768
 769         /* FIXME circumvention to improve performance when no irq is
 770          * there.
 771          */
 772         if (GENWQE_POLLING_ENABLED)
 773                 genwqe_check_ddcb_queue(cd, queue);
 774
 775         /*
 776          * It must be ensured to process all DDCBs in successive
 777          * order. Use a lock here in order to prevent nested DDCB
 778          * enqueuing.
 779          */
 780         spin_lock_irqsave(&queue->ddcb_lock, flags);
 781
 782         pddcb = get_next_ddcb(cd, queue, &req->num);    /* get ptr and num */
 783         if (pddcb == NULL) {
 784                 int rc;
 785
 786                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 787
 788                 if (f_flags & O_NONBLOCK) {
 789                         queue->return_on_busy++;
 790                         return -EBUSY;
 791                 }
 792
 793                 queue->wait_on_busy++;
 794                 rc = wait_event_interruptible(queue->busy_waitq,
 795                                               queue_free_ddcbs(queue) != 0);
 796                 dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n",
 797                         __func__, rc);
 798                 if (rc == -ERESTARTSYS)
 799                         return rc;  /* interrupted by a signal */
 800
 801                 goto retry;
 802         }
 803
 804         if (queue->ddcb_req[req->num] != NULL) {
 805                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 806
 807                 dev_err(&pci_dev->dev,
 808                         "[%s] picked DDCB %d with req=%p still in use!!\n",
 809                         __func__, req->num, req);
 810                 return -EFAULT;
 811         }
 812         ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED);
 813         queue->ddcb_req[req->num] = req;
 814
 815         pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts);
 816         pddcb->cmd = req->cmd.cmd;
 817         pddcb->acfunc = req->cmd.acfunc;        /* functional unit */
 818
 819         /*
 820          * We know that we can get retc 0x104 with CRC error, do not
 821          * stop the queue in those cases for this command. XDIR = 1
 822          * does not work for old SLU versions.
 823          *
 824          * Last bitstream with the old XDIR behavior had SLU_ID
 825          * 0x34199.
 826          */
 827         if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull)
 828                 pddcb->xdir = 0x1;
 829         else
 830                 pddcb->xdir = 0x0;
 831
 832
 833         pddcb->psp = (((req->cmd.asiv_length / 8) << 4) |
 834                       ((req->cmd.asv_length  / 8)));
 835         pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts);
 836
 837         /*
 838          * If copying the whole DDCB_ASIV_LENGTH is impacting
 839          * performance we need to change it to
 840          * req->cmd.asiv_length. But simulation benefits from some
 841          * non-architectured bits behind the architectured content.
 842          *
 843          * How much data is copied depends on the availability of the
 844          * ATS field, which was introduced late. If the ATS field is
 845          * supported ASIV is 8 bytes shorter than it used to be. Since
 846          * the ATS field is copied too, the code should do exactly
 847          * what it did before, but I wanted to make copying of the ATS
 848          * field very explicit.
 849          */
 850         if (genwqe_get_slu_id(cd) <= 0x2) {
 851                 memcpy(&pddcb->__asiv[0],       /* destination */
 852                        &req->cmd.__asiv[0],     /* source */
 853                        DDCB_ASIV_LENGTH);       /* req->cmd.asiv_length */
 854         } else {
 855                 pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats);
 856                 memcpy(&pddcb->n.asiv[0],       /* destination */
 857                         &req->cmd.asiv[0],      /* source */
 858                         DDCB_ASIV_LENGTH_ATS);  /* req->cmd.asiv_length */
 859         }
 860
 861         pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */
 862
 863         /*
 864          * Calculate CRC_16 for corresponding range PSP(7:4). Include
 865          * empty 4 bytes prior to the data.
 866          */
 867         icrc = genwqe_crc16((const u8 *)pddcb,
 868                            ICRC_LENGTH(req->cmd.asiv_length), 0xffff);
 869         pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16);
 870
 871         /* enable DDCB completion irq */
 872         if (!GENWQE_POLLING_ENABLED)
 873                 pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32;
 874
 875         dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num);
 876         genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 877
 878         if (ddcb_requ_collect_debug_data(req)) {
 879                 /* use the kernel copy of debug data. copying back to
 880                    user buffer happens later */
 881
 882                 genwqe_init_debug_data(cd, &req->debug_data);
 883                 memcpy(&req->debug_data.ddcb_before, pddcb,
 884                        sizeof(req->debug_data.ddcb_before));
 885         }
 886
 887         enqueue_ddcb(cd, queue, pddcb, req->num);
 888         queue->ddcbs_in_flight++;
 889
 890         if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight)
 891                 queue->ddcbs_max_in_flight = queue->ddcbs_in_flight;
 892
 893         ddcb_requ_set_state(req, GENWQE_REQU_TAPPED);
 894         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 895         wake_up_interruptible(&cd->queue_waitq);
 896
 897         return 0;
 898 }
 899
 900 /**
 901  * __genwqe_execute_raw_ddcb() - Setup and execute DDCB
 902  * @cd:         pointer to genwqe device descriptor
 903  * @req:        user provided DDCB request
 904  * @f_flags:    file mode: blocking, non-blocking
 905  */
 906 int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd,
 907                               struct genwqe_ddcb_cmd *cmd,
 908                               unsigned int f_flags)
 909 {
 910         int rc = 0;
 911         struct pci_dev *pci_dev = cd->pci_dev;
 912         struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
 913
 914         if (cmd->asiv_length > DDCB_ASIV_LENGTH) {
 915                 dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n",
 916                         __func__, cmd->asiv_length);
 917                 return -EINVAL;
 918         }
 919         if (cmd->asv_length > DDCB_ASV_LENGTH) {
 920                 dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n",
 921                         __func__, cmd->asiv_length);
 922                 return -EINVAL;
 923         }
 924         rc = __genwqe_enqueue_ddcb(cd, req, f_flags);
 925         if (rc != 0)
 926                 return rc;
 927
 928         rc = __genwqe_wait_ddcb(cd, req);
 929         if (rc < 0)             /* error or signal interrupt */
 930                 goto err_exit;
 931
 932         if (ddcb_requ_collect_debug_data(req)) {
 933                 if (copy_to_user((struct genwqe_debug_data __user *)
 934                                  (unsigned long)cmd->ddata_addr,
 935                                  &req->debug_data,
 936                                  sizeof(struct genwqe_debug_data)))
 937                         return -EFAULT;
 938         }
 939
 940         /*
 941          * Higher values than 0x102 indicate completion with faults,
 942          * lower values than 0x102 indicate processing faults. Note
 943          * that DDCB might have been purged. E.g. Cntl+C.
 944          */
 945         if (cmd->retc != DDCB_RETC_COMPLETE) {
 946                 /* This might happen e.g. flash read, and needs to be
 947                    handled by the upper layer code. */
 948                 rc = -EBADMSG;  /* not processed/error retc */
 949         }
 950
 951         return rc;
 952
 953  err_exit:
 954         __genwqe_purge_ddcb(cd, req);
 955
 956         if (ddcb_requ_collect_debug_data(req)) {
 957                 if (copy_to_user((struct genwqe_debug_data __user *)
 958                                  (unsigned long)cmd->ddata_addr,
 959                                  &req->debug_data,
 960                                  sizeof(struct genwqe_debug_data)))
 961                         return -EFAULT;
 962         }
 963         return rc;
 964 }
 965
 966 /**
 967  * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished
 968  *
 969  * We use this as condition for our wait-queue code.
 970  */
 971 static int genwqe_next_ddcb_ready(struct genwqe_dev *cd)
 972 {
 973         unsigned long flags;
 974         struct ddcb *pddcb;
 975         struct ddcb_queue *queue = &cd->queue;
 976
 977         spin_lock_irqsave(&queue->ddcb_lock, flags);
 978
 979         if (queue_empty(queue)) { /* emtpy queue */
 980                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 981                 return 0;
 982         }
 983
 984         pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
 985         if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */
 986                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 987                 return 1;
 988         }
 989
 990         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 991         return 0;
 992 }
 993
 994 /**
 995  * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight
 996  *
 997  * Keep track on the number of DDCBs which ware currently in the
 998  * queue. This is needed for statistics as well as conditon if we want
 999  * to wait or better do polling in case of no interrupts available.
1000  */
1001 int genwqe_ddcbs_in_flight(struct genwqe_dev *cd)
1002 {
1003         unsigned long flags;
1004         int ddcbs_in_flight = 0;
1005         struct ddcb_queue *queue = &cd->queue;
1006
1007         spin_lock_irqsave(&queue->ddcb_lock, flags);
1008         ddcbs_in_flight += queue->ddcbs_in_flight;
1009         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1010
1011         return ddcbs_in_flight;
1012 }
1013
1014 static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1015 {
1016         int rc, i;
1017         struct ddcb *pddcb;
1018         u64 val64;
1019         unsigned int queue_size;
1020         struct pci_dev *pci_dev = cd->pci_dev;
1021
1022         if (GENWQE_DDCB_MAX < 2)
1023                 return -EINVAL;
1024
1025         queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE);
1026
1027         queue->ddcbs_in_flight = 0;  /* statistics */
1028         queue->ddcbs_max_in_flight = 0;
1029         queue->ddcbs_completed = 0;
1030         queue->return_on_busy = 0;
1031         queue->wait_on_busy = 0;
1032
1033         queue->ddcb_seq   = 0x100; /* start sequence number */
1034         queue->ddcb_max   = GENWQE_DDCB_MAX;
1035         queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,
1036                                                 &queue->ddcb_daddr);
1037         if (queue->ddcb_vaddr == NULL) {
1038                 dev_err(&pci_dev->dev,
1039                         "[%s] **err: could not allocate DDCB **\n", __func__);
1040                 return -ENOMEM;
1041         }
1042         queue->ddcb_req = kcalloc(queue->ddcb_max, sizeof(struct ddcb_requ *),
1043                                   GFP_KERNEL);
1044         if (!queue->ddcb_req) {
1045                 rc = -ENOMEM;
1046                 goto free_ddcbs;
1047         }
1048
1049         queue->ddcb_waitqs = kcalloc(queue->ddcb_max,
1050                                      sizeof(wait_queue_head_t),
1051                                      GFP_KERNEL);
1052         if (!queue->ddcb_waitqs) {
1053                 rc = -ENOMEM;
1054                 goto free_requs;
1055         }
1056
1057         for (i = 0; i < queue->ddcb_max; i++) {
1058                 pddcb = &queue->ddcb_vaddr[i];               /* DDCBs */
1059                 pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32;
1060                 pddcb->retc_16 = cpu_to_be16(0xfff);
1061
1062                 queue->ddcb_req[i] = NULL;                   /* requests */
1063                 init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */
1064         }
1065
1066         queue->ddcb_act  = 0;
1067         queue->ddcb_next = 0;   /* queue is empty */
1068
1069         spin_lock_init(&queue->ddcb_lock);
1070         init_waitqueue_head(&queue->busy_waitq);
1071
1072         val64 = ((u64)(queue->ddcb_max - 1) <<  8); /* lastptr */
1073         __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG,  0x07);  /* iCRC/vCRC */
1074         __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr);
1075         __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq);
1076         __genwqe_writeq(cd, queue->IO_QUEUE_WRAP,    val64);
1077         return 0;
1078
1079  free_requs:
1080         kfree(queue->ddcb_req);
1081         queue->ddcb_req = NULL;
1082  free_ddcbs:
1083         __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1084                                 queue->ddcb_daddr);
1085         queue->ddcb_vaddr = NULL;
1086         queue->ddcb_daddr = 0ull;
1087         return -ENODEV;
1088
1089 }
1090
1091 static int ddcb_queue_initialized(struct ddcb_queue *queue)
1092 {
1093         return queue->ddcb_vaddr != NULL;
1094 }
1095
1096 static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1097 {
1098         unsigned int queue_size;
1099
1100         queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
1101
1102         kfree(queue->ddcb_req);
1103         queue->ddcb_req = NULL;
1104
1105         if (queue->ddcb_vaddr) {
1106                 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1107                                         queue->ddcb_daddr);
1108                 queue->ddcb_vaddr = NULL;
1109                 queue->ddcb_daddr = 0ull;
1110         }
1111 }
1112
1113 static irqreturn_t genwqe_pf_isr(int irq, void *dev_id)
1114 {
1115         u64 gfir;
1116         struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1117         struct pci_dev *pci_dev = cd->pci_dev;
1118
1119         /*
1120          * In case of fatal FIR error the queue is stopped, such that
1121          * we can safely check it without risking anything.
1122          */
1123         cd->irqs_processed++;
1124         wake_up_interruptible(&cd->queue_waitq);
1125
1126         /*
1127          * Checking for errors before kicking the queue might be
1128          * safer, but slower for the good-case ... See above.
1129          */
1130         gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
1131         if (((gfir & GFIR_ERR_TRIGGER) != 0x0) &&
1132             !pci_channel_offline(pci_dev)) {
1133
1134                 if (cd->use_platform_recovery) {
1135                         /*
1136                          * Since we use raw accessors, EEH errors won't be
1137                          * detected by the platform until we do a non-raw
1138                          * MMIO or config space read
1139                          */
1140                         readq(cd->mmio + IO_SLC_CFGREG_GFIR);
1141
1142                         /* Don't do anything if the PCI channel is frozen */
1143                         if (pci_channel_offline(pci_dev))
1144                                 goto exit;
1145                 }
1146
1147                 wake_up_interruptible(&cd->health_waitq);
1148
1149                 /*
1150                  * By default GFIRs causes recovery actions. This
1151                  * count is just for debug when recovery is masked.
1152                  */
1153                 dev_err_ratelimited(&pci_dev->dev,
1154                                     "[%s] GFIR=%016llx\n",
1155                                     __func__, gfir);
1156         }
1157
1158  exit:
1159         return IRQ_HANDLED;
1160 }
1161
1162 static irqreturn_t genwqe_vf_isr(int irq, void *dev_id)
1163 {
1164         struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1165
1166         cd->irqs_processed++;
1167         wake_up_interruptible(&cd->queue_waitq);
1168
1169         return IRQ_HANDLED;
1170 }
1171
1172 /**
1173  * genwqe_card_thread() - Work thread for the DDCB queue
1174  *
1175  * The idea is to check if there are DDCBs in processing. If there are
1176  * some finished DDCBs, we process them and wakeup the
1177  * requestors. Otherwise we give other processes time using
1178  * cond_resched().
1179  */
1180 static int genwqe_card_thread(void *data)
1181 {
1182         int should_stop = 0, rc = 0;
1183         struct genwqe_dev *cd = (struct genwqe_dev *)data;
1184
1185         while (!kthread_should_stop()) {
1186
1187                 genwqe_check_ddcb_queue(cd, &cd->queue);
1188
1189                 if (GENWQE_POLLING_ENABLED) {
1190                         rc = wait_event_interruptible_timeout(
1191                                 cd->queue_waitq,
1192                                 genwqe_ddcbs_in_flight(cd) ||
1193                                 (should_stop = kthread_should_stop()), 1);
1194                 } else {
1195                         rc = wait_event_interruptible_timeout(
1196                                 cd->queue_waitq,
1197                                 genwqe_next_ddcb_ready(cd) ||
1198                                 (should_stop = kthread_should_stop()), HZ);
1199                 }
1200                 if (should_stop)
1201                         break;
1202
1203                 /*
1204                  * Avoid soft lockups on heavy loads; we do not want
1205                  * to disable our interrupts.
1206                  */
1207                 cond_resched();
1208         }
1209         return 0;
1210 }
1211
1212 /**
1213  * genwqe_setup_service_layer() - Setup DDCB queue
1214  * @cd:         pointer to genwqe device descriptor
1215  *
1216  * Allocate DDCBs. Configure Service Layer Controller (SLC).
1217  *
1218  * Return: 0 success
1219  */
1220 int genwqe_setup_service_layer(struct genwqe_dev *cd)
1221 {
1222         int rc;
1223         struct ddcb_queue *queue;
1224         struct pci_dev *pci_dev = cd->pci_dev;
1225
1226         if (genwqe_is_privileged(cd)) {
1227                 rc = genwqe_card_reset(cd);
1228                 if (rc < 0) {
1229                         dev_err(&pci_dev->dev,
1230                                 "[%s] err: reset failed.\n", __func__);
1231                         return rc;
1232                 }
1233                 genwqe_read_softreset(cd);
1234         }
1235
1236         queue = &cd->queue;
1237         queue->IO_QUEUE_CONFIG  = IO_SLC_QUEUE_CONFIG;
1238         queue->IO_QUEUE_STATUS  = IO_SLC_QUEUE_STATUS;
1239         queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT;
1240         queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN;
1241         queue->IO_QUEUE_OFFSET  = IO_SLC_QUEUE_OFFSET;
1242         queue->IO_QUEUE_WRAP    = IO_SLC_QUEUE_WRAP;
1243         queue->IO_QUEUE_WTIME   = IO_SLC_QUEUE_WTIME;
1244         queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS;
1245         queue->IO_QUEUE_LRW     = IO_SLC_QUEUE_LRW;
1246
1247         rc = setup_ddcb_queue(cd, queue);
1248         if (rc != 0) {
1249                 rc = -ENODEV;
1250                 goto err_out;
1251         }
1252
1253         init_waitqueue_head(&cd->queue_waitq);
1254         cd->card_thread = kthread_run(genwqe_card_thread, cd,
1255                                       GENWQE_DEVNAME "%d_thread",
1256                                       cd->card_idx);
1257         if (IS_ERR(cd->card_thread)) {
1258                 rc = PTR_ERR(cd->card_thread);
1259                 cd->card_thread = NULL;
1260                 goto stop_free_queue;
1261         }
1262
1263         rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS);
1264         if (rc)
1265                 goto stop_kthread;
1266
1267         /*
1268          * We must have all wait-queues initialized when we enable the
1269          * interrupts. Otherwise we might crash if we get an early
1270          * irq.
1271          */
1272         init_waitqueue_head(&cd->health_waitq);
1273
1274         if (genwqe_is_privileged(cd)) {
1275                 rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED,
1276                                  GENWQE_DEVNAME, cd);
1277         } else {
1278                 rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED,
1279                                  GENWQE_DEVNAME, cd);
1280         }
1281         if (rc < 0) {
1282                 dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq);
1283                 goto stop_irq_cap;
1284         }
1285
1286         cd->card_state = GENWQE_CARD_USED;
1287         return 0;
1288
1289  stop_irq_cap:
1290         genwqe_reset_interrupt_capability(cd);
1291  stop_kthread:
1292         kthread_stop(cd->card_thread);
1293         cd->card_thread = NULL;
1294  stop_free_queue:
1295         free_ddcb_queue(cd, queue);
1296  err_out:
1297         return rc;
1298 }
1299
1300 /**
1301  * queue_wake_up_all() - Handles fatal error case
1302  *
1303  * The PCI device got unusable and we have to stop all pending
1304  * requests as fast as we can. The code after this must purge the
1305  * DDCBs in question and ensure that all mappings are freed.
1306  */
1307 static int queue_wake_up_all(struct genwqe_dev *cd)
1308 {
1309         unsigned int i;
1310         unsigned long flags;
1311         struct ddcb_queue *queue = &cd->queue;
1312
1313         spin_lock_irqsave(&queue->ddcb_lock, flags);
1314
1315         for (i = 0; i < queue->ddcb_max; i++)
1316                 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
1317
1318         wake_up_interruptible(&queue->busy_waitq);
1319         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1320
1321         return 0;
1322 }
1323
1324 /**
1325  * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces
1326  *
1327  * Relies on the pre-condition that there are no users of the card
1328  * device anymore e.g. with open file-descriptors.
1329  *
1330  * This function must be robust enough to be called twice.
1331  */
1332 int genwqe_finish_queue(struct genwqe_dev *cd)
1333 {
1334         int i, rc = 0, in_flight;
1335         int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT;
1336         struct pci_dev *pci_dev = cd->pci_dev;
1337         struct ddcb_queue *queue = &cd->queue;
1338
1339         if (!ddcb_queue_initialized(queue))
1340                 return 0;
1341
1342         /* Do not wipe out the error state. */
1343         if (cd->card_state == GENWQE_CARD_USED)
1344                 cd->card_state = GENWQE_CARD_UNUSED;
1345
1346         /* Wake up all requests in the DDCB queue such that they
1347            should be removed nicely. */
1348         queue_wake_up_all(cd);
1349
1350         /* We must wait to get rid of the DDCBs in flight */
1351         for (i = 0; i < waitmax; i++) {
1352                 in_flight = genwqe_ddcbs_in_flight(cd);
1353
1354                 if (in_flight == 0)
1355                         break;
1356
1357                 dev_dbg(&pci_dev->dev,
1358                         "  DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n",
1359                         i, waitmax, in_flight);
1360
1361                 /*
1362                  * Severe severe error situation: The card itself has
1363                  * 16 DDCB queues, each queue has e.g. 32 entries,
1364                  * each DDBC has a hardware timeout of currently 250
1365                  * msec but the PFs have a hardware timeout of 8 sec
1366                  * ... so I take something large.
1367                  */
1368                 msleep(1000);
1369         }
1370         if (i == waitmax) {
1371                 dev_err(&pci_dev->dev, "  [%s] err: queue is not empty!!\n",
1372                         __func__);
1373                 rc = -EIO;
1374         }
1375         return rc;
1376 }
1377
1378 /**
1379  * genwqe_release_service_layer() - Shutdown DDCB queue
1380  * @cd:       genwqe device descriptor
1381  *
1382  * This function must be robust enough to be called twice.
1383  */
1384 int genwqe_release_service_layer(struct genwqe_dev *cd)
1385 {
1386         struct pci_dev *pci_dev = cd->pci_dev;
1387
1388         if (!ddcb_queue_initialized(&cd->queue))
1389                 return 1;
1390
1391         free_irq(pci_dev->irq, cd);
1392         genwqe_reset_interrupt_capability(cd);
1393
1394         if (cd->card_thread != NULL) {
1395                 kthread_stop(cd->card_thread);
1396                 cd->card_thread = NULL;
1397         }
1398
1399         free_ddcb_queue(cd, &cd->queue);
1400         return 0;
1401 }