tools/io_uring/io_uring-bench.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Simple benchmark program that uses the various features of io_uring
   4  * to provide fast random access to a device/file. It has various
   5  * options that are control how we use io_uring, see the OPTIONS section
   6  * below. This uses the raw io_uring interface.
   7  *
   8  * Copyright (C) 2018-2019 Jens Axboe
   9  */
  10 #include <stdio.h>
  11 #include <errno.h>
  12 #include <assert.h>
  13 #include <stdlib.h>
  14 #include <stddef.h>
  15 #include <signal.h>
  16 #include <inttypes.h>
  17
  18 #include <sys/types.h>
  19 #include <sys/stat.h>
  20 #include <sys/ioctl.h>
  21 #include <sys/syscall.h>
  22 #include <sys/resource.h>
  23 #include <sys/mman.h>
  24 #include <sys/uio.h>
  25 #include <linux/fs.h>
  26 #include <fcntl.h>
  27 #include <unistd.h>
  28 #include <string.h>
  29 #include <pthread.h>
  30 #include <sched.h>
  31
  32 #include "liburing.h"
  33 #include "barrier.h"
  34
  35 #ifndef IOCQE_FLAG_CACHEHIT
  36 #define IOCQE_FLAG_CACHEHIT     (1U << 0)
  37 #endif
  38
  39 #define min(a, b)               ((a < b) ? (a) : (b))
  40
  41 struct io_sq_ring {
  42         unsigned *head;
  43         unsigned *tail;
  44         unsigned *ring_mask;
  45         unsigned *ring_entries;
  46         unsigned *flags;
  47         unsigned *array;
  48 };
  49
  50 struct io_cq_ring {
  51         unsigned *head;
  52         unsigned *tail;
  53         unsigned *ring_mask;
  54         unsigned *ring_entries;
  55         struct io_uring_cqe *cqes;
  56 };
  57
  58 #define DEPTH                   128
  59
  60 #define BATCH_SUBMIT            32
  61 #define BATCH_COMPLETE          32
  62
  63 #define BS                      4096
  64
  65 #define MAX_FDS                 16
  66
  67 static unsigned sq_ring_mask, cq_ring_mask;
  68
  69 struct file {
  70         unsigned long max_blocks;
  71         unsigned pending_ios;
  72         int real_fd;
  73         int fixed_fd;
  74 };
  75
  76 struct submitter {
  77         pthread_t thread;
  78         int ring_fd;
  79         struct drand48_data rand;
  80         struct io_sq_ring sq_ring;
  81         struct io_uring_sqe *sqes;
  82         struct iovec iovecs[DEPTH];
  83         struct io_cq_ring cq_ring;
  84         int inflight;
  85         unsigned long reaps;
  86         unsigned long done;
  87         unsigned long calls;
  88         unsigned long cachehit, cachemiss;
  89         volatile int finish;
  90
  91         __s32 *fds;
  92
  93         struct file files[MAX_FDS];
  94         unsigned nr_files;
  95         unsigned cur_file;
  96 };
  97
  98 static struct submitter submitters[1];
  99 static volatile int finish;
 100
 101 /*
 102  * OPTIONS: Set these to test the various features of io_uring.
 103  */
 104 static int polled = 1;          /* use IO polling */
 105 static int fixedbufs = 1;       /* use fixed user buffers */
 106 static int register_files = 1;  /* use fixed files */
 107 static int buffered = 0;        /* use buffered IO, not O_DIRECT */
 108 static int sq_thread_poll = 0;  /* use kernel submission/poller thread */
 109 static int sq_thread_cpu = -1;  /* pin above thread to this CPU */
 110 static int do_nop = 0;          /* no-op SQ ring commands */
 111
 112 static int io_uring_register_buffers(struct submitter *s)
 113 {
 114         if (do_nop)
 115                 return 0;
 116
 117         return io_uring_register(s->ring_fd, IORING_REGISTER_BUFFERS, s->iovecs,
 118                                         DEPTH);
 119 }
 120
 121 static int io_uring_register_files(struct submitter *s)
 122 {
 123         unsigned i;
 124
 125         if (do_nop)
 126                 return 0;
 127
 128         s->fds = calloc(s->nr_files, sizeof(__s32));
 129         for (i = 0; i < s->nr_files; i++) {
 130                 s->fds[i] = s->files[i].real_fd;
 131                 s->files[i].fixed_fd = i;
 132         }
 133
 134         return io_uring_register(s->ring_fd, IORING_REGISTER_FILES, s->fds,
 135                                         s->nr_files);
 136 }
 137
 138 static int gettid(void)
 139 {
 140         return syscall(__NR_gettid);
 141 }
 142
 143 static unsigned file_depth(struct submitter *s)
 144 {
 145         return (DEPTH + s->nr_files - 1) / s->nr_files;
 146 }
 147
 148 static void init_io(struct submitter *s, unsigned index)
 149 {
 150         struct io_uring_sqe *sqe = &s->sqes[index];
 151         unsigned long offset;
 152         struct file *f;
 153         long r;
 154
 155         if (do_nop) {
 156                 sqe->opcode = IORING_OP_NOP;
 157                 return;
 158         }
 159
 160         if (s->nr_files == 1) {
 161                 f = &s->files[0];
 162         } else {
 163                 f = &s->files[s->cur_file];
 164                 if (f->pending_ios >= file_depth(s)) {
 165                         s->cur_file++;
 166                         if (s->cur_file == s->nr_files)
 167                                 s->cur_file = 0;
 168                         f = &s->files[s->cur_file];
 169                 }
 170         }
 171         f->pending_ios++;
 172
 173         lrand48_r(&s->rand, &r);
 174         offset = (r % (f->max_blocks - 1)) * BS;
 175
 176         if (register_files) {
 177                 sqe->flags = IOSQE_FIXED_FILE;
 178                 sqe->fd = f->fixed_fd;
 179         } else {
 180                 sqe->flags = 0;
 181                 sqe->fd = f->real_fd;
 182         }
 183         if (fixedbufs) {
 184                 sqe->opcode = IORING_OP_READ_FIXED;
 185                 sqe->addr = (unsigned long) s->iovecs[index].iov_base;
 186                 sqe->len = BS;
 187                 sqe->buf_index = index;
 188         } else {
 189                 sqe->opcode = IORING_OP_READV;
 190                 sqe->addr = (unsigned long) &s->iovecs[index];
 191                 sqe->len = 1;
 192                 sqe->buf_index = 0;
 193         }
 194         sqe->ioprio = 0;
 195         sqe->off = offset;
 196         sqe->user_data = (unsigned long) f;
 197 }
 198
 199 static int prep_more_ios(struct submitter *s, unsigned max_ios)
 200 {
 201         struct io_sq_ring *ring = &s->sq_ring;
 202         unsigned index, tail, next_tail, prepped = 0;
 203
 204         next_tail = tail = *ring->tail;
 205         do {
 206                 next_tail++;
 207                 read_barrier();
 208                 if (next_tail == *ring->head)
 209                         break;
 210
 211                 index = tail & sq_ring_mask;
 212                 init_io(s, index);
 213                 ring->array[index] = index;
 214                 prepped++;
 215                 tail = next_tail;
 216         } while (prepped < max_ios);
 217
 218         if (*ring->tail != tail) {
 219                 /* order tail store with writes to sqes above */
 220                 write_barrier();
 221                 *ring->tail = tail;
 222                 write_barrier();
 223         }
 224         return prepped;
 225 }
 226
 227 static int get_file_size(struct file *f)
 228 {
 229         struct stat st;
 230
 231         if (fstat(f->real_fd, &st) < 0)
 232                 return -1;
 233         if (S_ISBLK(st.st_mode)) {
 234                 unsigned long long bytes;
 235
 236                 if (ioctl(f->real_fd, BLKGETSIZE64, &bytes) != 0)
 237                         return -1;
 238
 239                 f->max_blocks = bytes / BS;
 240                 return 0;
 241         } else if (S_ISREG(st.st_mode)) {
 242                 f->max_blocks = st.st_size / BS;
 243                 return 0;
 244         }
 245
 246         return -1;
 247 }
 248
 249 static int reap_events(struct submitter *s)
 250 {
 251         struct io_cq_ring *ring = &s->cq_ring;
 252         struct io_uring_cqe *cqe;
 253         unsigned head, reaped = 0;
 254
 255         head = *ring->head;
 256         do {
 257                 struct file *f;
 258
 259                 read_barrier();
 260                 if (head == *ring->tail)
 261                         break;
 262                 cqe = &ring->cqes[head & cq_ring_mask];
 263                 if (!do_nop) {
 264                         f = (struct file *) (uintptr_t) cqe->user_data;
 265                         f->pending_ios--;
 266                         if (cqe->res != BS) {
 267                                 printf("io: unexpected ret=%d\n", cqe->res);
 268                                 if (polled && cqe->res == -EOPNOTSUPP)
 269                                         printf("Your filesystem doesn't support poll\n");
 270                                 return -1;
 271                         }
 272                 }
 273                 if (cqe->flags & IOCQE_FLAG_CACHEHIT)
 274                         s->cachehit++;
 275                 else
 276                         s->cachemiss++;
 277                 reaped++;
 278                 head++;
 279         } while (1);
 280
 281         s->inflight -= reaped;
 282         *ring->head = head;
 283         write_barrier();
 284         return reaped;
 285 }
 286
 287 static void *submitter_fn(void *data)
 288 {
 289         struct submitter *s = data;
 290         struct io_sq_ring *ring = &s->sq_ring;
 291         int ret, prepped;
 292
 293         printf("submitter=%d\n", gettid());
 294
 295         srand48_r(pthread_self(), &s->rand);
 296
 297         prepped = 0;
 298         do {
 299                 int to_wait, to_submit, this_reap, to_prep;
 300
 301                 if (!prepped && s->inflight < DEPTH) {
 302                         to_prep = min(DEPTH - s->inflight, BATCH_SUBMIT);
 303                         prepped = prep_more_ios(s, to_prep);
 304                 }
 305                 s->inflight += prepped;
 306 submit_more:
 307                 to_submit = prepped;
 308 submit:
 309                 if (to_submit && (s->inflight + to_submit <= DEPTH))
 310                         to_wait = 0;
 311                 else
 312                         to_wait = min(s->inflight + to_submit, BATCH_COMPLETE);
 313
 314                 /*
 315                  * Only need to call io_uring_enter if we're not using SQ thread
 316                  * poll, or if IORING_SQ_NEED_WAKEUP is set.
 317                  */
 318                 if (!sq_thread_poll || (*ring->flags & IORING_SQ_NEED_WAKEUP)) {
 319                         unsigned flags = 0;
 320
 321                         if (to_wait)
 322                                 flags = IORING_ENTER_GETEVENTS;
 323                         if ((*ring->flags & IORING_SQ_NEED_WAKEUP))
 324                                 flags |= IORING_ENTER_SQ_WAKEUP;
 325                         ret = io_uring_enter(s->ring_fd, to_submit, to_wait,
 326                                                 flags, NULL);
 327                         s->calls++;
 328                 }
 329
 330                 /*
 331                  * For non SQ thread poll, we already got the events we needed
 332                  * through the io_uring_enter() above. For SQ thread poll, we
 333                  * need to loop here until we find enough events.
 334                  */
 335                 this_reap = 0;
 336                 do {
 337                         int r;
 338                         r = reap_events(s);
 339                         if (r == -1) {
 340                                 s->finish = 1;
 341                                 break;
 342                         } else if (r > 0)
 343                                 this_reap += r;
 344                 } while (sq_thread_poll && this_reap < to_wait);
 345                 s->reaps += this_reap;
 346
 347                 if (ret >= 0) {
 348                         if (!ret) {
 349                                 to_submit = 0;
 350                                 if (s->inflight)
 351                                         goto submit;
 352                                 continue;
 353                         } else if (ret < to_submit) {
 354                                 int diff = to_submit - ret;
 355
 356                                 s->done += ret;
 357                                 prepped -= diff;
 358                                 goto submit_more;
 359                         }
 360                         s->done += ret;
 361                         prepped = 0;
 362                         continue;
 363                 } else if (ret < 0) {
 364                         if (errno == EAGAIN) {
 365                                 if (s->finish)
 366                                         break;
 367                                 if (this_reap)
 368                                         goto submit;
 369                                 to_submit = 0;
 370                                 goto submit;
 371                         }
 372                         printf("io_submit: %s\n", strerror(errno));
 373                         break;
 374                 }
 375         } while (!s->finish);
 376
 377         finish = 1;
 378         return NULL;
 379 }
 380
 381 static void sig_int(int sig)
 382 {
 383         printf("Exiting on signal %d\n", sig);
 384         submitters[0].finish = 1;
 385         finish = 1;
 386 }
 387
 388 static void arm_sig_int(void)
 389 {
 390         struct sigaction act;
 391
 392         memset(&act, 0, sizeof(act));
 393         act.sa_handler = sig_int;
 394         act.sa_flags = SA_RESTART;
 395         sigaction(SIGINT, &act, NULL);
 396 }
 397
 398 static int setup_ring(struct submitter *s)
 399 {
 400         struct io_sq_ring *sring = &s->sq_ring;
 401         struct io_cq_ring *cring = &s->cq_ring;
 402         struct io_uring_params p;
 403         int ret, fd;
 404         void *ptr;
 405
 406         memset(&p, 0, sizeof(p));
 407
 408         if (polled && !do_nop)
 409                 p.flags |= IORING_SETUP_IOPOLL;
 410         if (sq_thread_poll) {
 411                 p.flags |= IORING_SETUP_SQPOLL;
 412                 if (sq_thread_cpu != -1) {
 413                         p.flags |= IORING_SETUP_SQ_AFF;
 414                         p.sq_thread_cpu = sq_thread_cpu;
 415                 }
 416         }
 417
 418         fd = io_uring_setup(DEPTH, &p);
 419         if (fd < 0) {
 420                 perror("io_uring_setup");
 421                 return 1;
 422         }
 423         s->ring_fd = fd;
 424
 425         if (fixedbufs) {
 426                 ret = io_uring_register_buffers(s);
 427                 if (ret < 0) {
 428                         perror("io_uring_register_buffers");
 429                         return 1;
 430                 }
 431         }
 432
 433         if (register_files) {
 434                 ret = io_uring_register_files(s);
 435                 if (ret < 0) {
 436                         perror("io_uring_register_files");
 437                         return 1;
 438                 }
 439         }
 440
 441         ptr = mmap(0, p.sq_off.array + p.sq_entries * sizeof(__u32),
 442                         PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
 443                         IORING_OFF_SQ_RING);
 444         printf("sq_ring ptr = 0x%p\n", ptr);
 445         sring->head = ptr + p.sq_off.head;
 446         sring->tail = ptr + p.sq_off.tail;
 447         sring->ring_mask = ptr + p.sq_off.ring_mask;
 448         sring->ring_entries = ptr + p.sq_off.ring_entries;
 449         sring->flags = ptr + p.sq_off.flags;
 450         sring->array = ptr + p.sq_off.array;
 451         sq_ring_mask = *sring->ring_mask;
 452
 453         s->sqes = mmap(0, p.sq_entries * sizeof(struct io_uring_sqe),
 454                         PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
 455                         IORING_OFF_SQES);
 456         printf("sqes ptr    = 0x%p\n", s->sqes);
 457
 458         ptr = mmap(0, p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe),
 459                         PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
 460                         IORING_OFF_CQ_RING);
 461         printf("cq_ring ptr = 0x%p\n", ptr);
 462         cring->head = ptr + p.cq_off.head;
 463         cring->tail = ptr + p.cq_off.tail;
 464         cring->ring_mask = ptr + p.cq_off.ring_mask;
 465         cring->ring_entries = ptr + p.cq_off.ring_entries;
 466         cring->cqes = ptr + p.cq_off.cqes;
 467         cq_ring_mask = *cring->ring_mask;
 468         return 0;
 469 }
 470
 471 static void file_depths(char *buf)
 472 {
 473         struct submitter *s = &submitters[0];
 474         unsigned i;
 475         char *p;
 476
 477         buf[0] = '\0';
 478         p = buf;
 479         for (i = 0; i < s->nr_files; i++) {
 480                 struct file *f = &s->files[i];
 481
 482                 if (i + 1 == s->nr_files)
 483                         p += sprintf(p, "%d", f->pending_ios);
 484                 else
 485                         p += sprintf(p, "%d, ", f->pending_ios);
 486         }
 487 }
 488
 489 int main(int argc, char *argv[])
 490 {
 491         struct submitter *s = &submitters[0];
 492         unsigned long done, calls, reap, cache_hit, cache_miss;
 493         int err, i, flags, fd;
 494         char *fdepths;
 495         void *ret;
 496
 497         if (!do_nop && argc < 2) {
 498                 printf("%s: filename\n", argv[0]);
 499                 return 1;
 500         }
 501
 502         flags = O_RDONLY | O_NOATIME;
 503         if (!buffered)
 504                 flags |= O_DIRECT;
 505
 506         i = 1;
 507         while (!do_nop && i < argc) {
 508                 struct file *f;
 509
 510                 if (s->nr_files == MAX_FDS) {
 511                         printf("Max number of files (%d) reached\n", MAX_FDS);
 512                         break;
 513                 }
 514                 fd = open(argv[i], flags);
 515                 if (fd < 0) {
 516                         perror("open");
 517                         return 1;
 518                 }
 519
 520                 f = &s->files[s->nr_files];
 521                 f->real_fd = fd;
 522                 if (get_file_size(f)) {
 523                         printf("failed getting size of device/file\n");
 524                         return 1;
 525                 }
 526                 if (f->max_blocks <= 1) {
 527                         printf("Zero file/device size?\n");
 528                         return 1;
 529                 }
 530                 f->max_blocks--;
 531
 532                 printf("Added file %s\n", argv[i]);
 533                 s->nr_files++;
 534                 i++;
 535         }
 536
 537         if (fixedbufs) {
 538                 struct rlimit rlim;
 539
 540                 rlim.rlim_cur = RLIM_INFINITY;
 541                 rlim.rlim_max = RLIM_INFINITY;
 542                 if (setrlimit(RLIMIT_MEMLOCK, &rlim) < 0) {
 543                         perror("setrlimit");
 544                         return 1;
 545                 }
 546         }
 547
 548         arm_sig_int();
 549
 550         for (i = 0; i < DEPTH; i++) {
 551                 void *buf;
 552
 553                 if (posix_memalign(&buf, BS, BS)) {
 554                         printf("failed alloc\n");
 555                         return 1;
 556                 }
 557                 s->iovecs[i].iov_base = buf;
 558                 s->iovecs[i].iov_len = BS;
 559         }
 560
 561         err = setup_ring(s);
 562         if (err) {
 563                 printf("ring setup failed: %s, %d\n", strerror(errno), err);
 564                 return 1;
 565         }
 566         printf("polled=%d, fixedbufs=%d, buffered=%d", polled, fixedbufs, buffered);
 567         printf(" QD=%d, sq_ring=%d, cq_ring=%d\n", DEPTH, *s->sq_ring.ring_entries, *s->cq_ring.ring_entries);
 568
 569         pthread_create(&s->thread, NULL, submitter_fn, s);
 570
 571         fdepths = malloc(8 * s->nr_files);
 572         cache_hit = cache_miss = reap = calls = done = 0;
 573         do {
 574                 unsigned long this_done = 0;
 575                 unsigned long this_reap = 0;
 576                 unsigned long this_call = 0;
 577                 unsigned long this_cache_hit = 0;
 578                 unsigned long this_cache_miss = 0;
 579                 unsigned long rpc = 0, ipc = 0;
 580                 double hit = 0.0;
 581
 582                 sleep(1);
 583                 this_done += s->done;
 584                 this_call += s->calls;
 585                 this_reap += s->reaps;
 586                 this_cache_hit += s->cachehit;
 587                 this_cache_miss += s->cachemiss;
 588                 if (this_cache_hit && this_cache_miss) {
 589                         unsigned long hits, total;
 590
 591                         hits = this_cache_hit - cache_hit;
 592                         total = hits + this_cache_miss - cache_miss;
 593                         hit = (double) hits / (double) total;
 594                         hit *= 100.0;
 595                 }
 596                 if (this_call - calls) {
 597                         rpc = (this_done - done) / (this_call - calls);
 598                         ipc = (this_reap - reap) / (this_call - calls);
 599                 } else
 600                         rpc = ipc = -1;
 601                 file_depths(fdepths);
 602                 printf("IOPS=%lu, IOS/call=%ld/%ld, inflight=%u (%s), Cachehit=%0.2f%%\n",
 603                                 this_done - done, rpc, ipc, s->inflight,
 604                                 fdepths, hit);
 605                 done = this_done;
 606                 calls = this_call;
 607                 reap = this_reap;
 608                 cache_hit = s->cachehit;
 609                 cache_miss = s->cachemiss;
 610         } while (!finish);
 611
 612         pthread_join(s->thread, &ret);
 613         close(s->ring_fd);
 614         free(fdepths);
 615         return 0;
 616 }