fs/fuse/virtio_fs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * virtio-fs: Virtio Filesystem
   4  * Copyright (C) 2018 Red Hat, Inc.
   5  */
   6
   7 #include <linux/fs.h>
   8 #include <linux/module.h>
   9 #include <linux/virtio.h>
  10 #include <linux/virtio_fs.h>
  11 #include <linux/delay.h>
  12 #include <linux/fs_context.h>
  13 #include <linux/highmem.h>
  14 #include "fuse_i.h"
  15
  16 /* List of virtio-fs device instances and a lock for the list. Also provides
  17  * mutual exclusion in device removal and mounting path
  18  */
  19 static DEFINE_MUTEX(virtio_fs_mutex);
  20 static LIST_HEAD(virtio_fs_instances);
  21
  22 enum {
  23         VQ_HIPRIO,
  24         VQ_REQUEST
  25 };
  26
  27 /* Per-virtqueue state */
  28 struct virtio_fs_vq {
  29         spinlock_t lock;
  30         struct virtqueue *vq;     /* protected by ->lock */
  31         struct work_struct done_work;
  32         struct list_head queued_reqs;
  33         struct list_head end_reqs;      /* End these requests */
  34         struct delayed_work dispatch_work;
  35         struct fuse_dev *fud;
  36         bool connected;
  37         long in_flight;
  38         char name[24];
  39 } ____cacheline_aligned_in_smp;
  40
  41 /* A virtio-fs device instance */
  42 struct virtio_fs {
  43         struct kref refcount;
  44         struct list_head list;    /* on virtio_fs_instances */
  45         char *tag;
  46         struct virtio_fs_vq *vqs;
  47         unsigned int nvqs;               /* number of virtqueues */
  48         unsigned int num_request_queues; /* number of request queues */
  49 };
  50
  51 struct virtio_fs_forget {
  52         struct fuse_in_header ih;
  53         struct fuse_forget_in arg;
  54         /* This request can be temporarily queued on virt queue */
  55         struct list_head list;
  56 };
  57
  58 static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq)
  59 {
  60         struct virtio_fs *fs = vq->vdev->priv;
  61
  62         return &fs->vqs[vq->index];
  63 }
  64
  65 static inline struct fuse_pqueue *vq_to_fpq(struct virtqueue *vq)
  66 {
  67         return &vq_to_fsvq(vq)->fud->pq;
  68 }
  69
  70 static void release_virtio_fs_obj(struct kref *ref)
  71 {
  72         struct virtio_fs *vfs = container_of(ref, struct virtio_fs, refcount);
  73
  74         kfree(vfs->vqs);
  75         kfree(vfs);
  76 }
  77
  78 /* Make sure virtiofs_mutex is held */
  79 static void virtio_fs_put(struct virtio_fs *fs)
  80 {
  81         kref_put(&fs->refcount, release_virtio_fs_obj);
  82 }
  83
  84 static void virtio_fs_fiq_release(struct fuse_iqueue *fiq)
  85 {
  86         struct virtio_fs *vfs = fiq->priv;
  87
  88         mutex_lock(&virtio_fs_mutex);
  89         virtio_fs_put(vfs);
  90         mutex_unlock(&virtio_fs_mutex);
  91 }
  92
  93 static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq)
  94 {
  95         WARN_ON(fsvq->in_flight < 0);
  96
  97         /* Wait for in flight requests to finish.*/
  98         while (1) {
  99                 spin_lock(&fsvq->lock);
 100                 if (!fsvq->in_flight) {
 101                         spin_unlock(&fsvq->lock);
 102                         break;
 103                 }
 104                 spin_unlock(&fsvq->lock);
 105                 /* TODO use completion instead of timeout */
 106                 usleep_range(1000, 2000);
 107         }
 108
 109         flush_work(&fsvq->done_work);
 110         flush_delayed_work(&fsvq->dispatch_work);
 111 }
 112
 113 static inline void drain_hiprio_queued_reqs(struct virtio_fs_vq *fsvq)
 114 {
 115         struct virtio_fs_forget *forget;
 116
 117         spin_lock(&fsvq->lock);
 118         while (1) {
 119                 forget = list_first_entry_or_null(&fsvq->queued_reqs,
 120                                                 struct virtio_fs_forget, list);
 121                 if (!forget)
 122                         break;
 123                 list_del(&forget->list);
 124                 kfree(forget);
 125         }
 126         spin_unlock(&fsvq->lock);
 127 }
 128
 129 static void virtio_fs_drain_all_queues(struct virtio_fs *fs)
 130 {
 131         struct virtio_fs_vq *fsvq;
 132         int i;
 133
 134         for (i = 0; i < fs->nvqs; i++) {
 135                 fsvq = &fs->vqs[i];
 136                 if (i == VQ_HIPRIO)
 137                         drain_hiprio_queued_reqs(fsvq);
 138
 139                 virtio_fs_drain_queue(fsvq);
 140         }
 141 }
 142
 143 static void virtio_fs_start_all_queues(struct virtio_fs *fs)
 144 {
 145         struct virtio_fs_vq *fsvq;
 146         int i;
 147
 148         for (i = 0; i < fs->nvqs; i++) {
 149                 fsvq = &fs->vqs[i];
 150                 spin_lock(&fsvq->lock);
 151                 fsvq->connected = true;
 152                 spin_unlock(&fsvq->lock);
 153         }
 154 }
 155
 156 /* Add a new instance to the list or return -EEXIST if tag name exists*/
 157 static int virtio_fs_add_instance(struct virtio_fs *fs)
 158 {
 159         struct virtio_fs *fs2;
 160         bool duplicate = false;
 161
 162         mutex_lock(&virtio_fs_mutex);
 163
 164         list_for_each_entry(fs2, &virtio_fs_instances, list) {
 165                 if (strcmp(fs->tag, fs2->tag) == 0)
 166                         duplicate = true;
 167         }
 168
 169         if (!duplicate)
 170                 list_add_tail(&fs->list, &virtio_fs_instances);
 171
 172         mutex_unlock(&virtio_fs_mutex);
 173
 174         if (duplicate)
 175                 return -EEXIST;
 176         return 0;
 177 }
 178
 179 /* Return the virtio_fs with a given tag, or NULL */
 180 static struct virtio_fs *virtio_fs_find_instance(const char *tag)
 181 {
 182         struct virtio_fs *fs;
 183
 184         mutex_lock(&virtio_fs_mutex);
 185
 186         list_for_each_entry(fs, &virtio_fs_instances, list) {
 187                 if (strcmp(fs->tag, tag) == 0) {
 188                         kref_get(&fs->refcount);
 189                         goto found;
 190                 }
 191         }
 192
 193         fs = NULL; /* not found */
 194
 195 found:
 196         mutex_unlock(&virtio_fs_mutex);
 197
 198         return fs;
 199 }
 200
 201 static void virtio_fs_free_devs(struct virtio_fs *fs)
 202 {
 203         unsigned int i;
 204
 205         for (i = 0; i < fs->nvqs; i++) {
 206                 struct virtio_fs_vq *fsvq = &fs->vqs[i];
 207
 208                 if (!fsvq->fud)
 209                         continue;
 210
 211                 fuse_dev_free(fsvq->fud);
 212                 fsvq->fud = NULL;
 213         }
 214 }
 215
 216 /* Read filesystem name from virtio config into fs->tag (must kfree()). */
 217 static int virtio_fs_read_tag(struct virtio_device *vdev, struct virtio_fs *fs)
 218 {
 219         char tag_buf[sizeof_field(struct virtio_fs_config, tag)];
 220         char *end;
 221         size_t len;
 222
 223         virtio_cread_bytes(vdev, offsetof(struct virtio_fs_config, tag),
 224                            &tag_buf, sizeof(tag_buf));
 225         end = memchr(tag_buf, '\0', sizeof(tag_buf));
 226         if (end == tag_buf)
 227                 return -EINVAL; /* empty tag */
 228         if (!end)
 229                 end = &tag_buf[sizeof(tag_buf)];
 230
 231         len = end - tag_buf;
 232         fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL);
 233         if (!fs->tag)
 234                 return -ENOMEM;
 235         memcpy(fs->tag, tag_buf, len);
 236         fs->tag[len] = '\0';
 237         return 0;
 238 }
 239
 240 /* Work function for hiprio completion */
 241 static void virtio_fs_hiprio_done_work(struct work_struct *work)
 242 {
 243         struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
 244                                                  done_work);
 245         struct virtqueue *vq = fsvq->vq;
 246
 247         /* Free completed FUSE_FORGET requests */
 248         spin_lock(&fsvq->lock);
 249         do {
 250                 unsigned int len;
 251                 void *req;
 252
 253                 virtqueue_disable_cb(vq);
 254
 255                 while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
 256                         kfree(req);
 257                         fsvq->in_flight--;
 258                 }
 259         } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
 260         spin_unlock(&fsvq->lock);
 261 }
 262
 263 static void virtio_fs_request_dispatch_work(struct work_struct *work)
 264 {
 265         struct fuse_req *req;
 266         struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
 267                                                  dispatch_work.work);
 268         struct fuse_conn *fc = fsvq->fud->fc;
 269
 270         pr_debug("virtio-fs: worker %s called.\n", __func__);
 271         while (1) {
 272                 spin_lock(&fsvq->lock);
 273                 req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req,
 274                                                list);
 275                 if (!req) {
 276                         spin_unlock(&fsvq->lock);
 277                         return;
 278                 }
 279
 280                 list_del_init(&req->list);
 281                 spin_unlock(&fsvq->lock);
 282                 fuse_request_end(fc, req);
 283         }
 284 }
 285
 286 static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
 287 {
 288         struct virtio_fs_forget *forget;
 289         struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
 290                                                  dispatch_work.work);
 291         struct virtqueue *vq = fsvq->vq;
 292         struct scatterlist sg;
 293         struct scatterlist *sgs[] = {&sg};
 294         bool notify;
 295         int ret;
 296
 297         pr_debug("virtio-fs: worker %s called.\n", __func__);
 298         while (1) {
 299                 spin_lock(&fsvq->lock);
 300                 forget = list_first_entry_or_null(&fsvq->queued_reqs,
 301                                         struct virtio_fs_forget, list);
 302                 if (!forget) {
 303                         spin_unlock(&fsvq->lock);
 304                         return;
 305                 }
 306
 307                 list_del(&forget->list);
 308                 if (!fsvq->connected) {
 309                         spin_unlock(&fsvq->lock);
 310                         kfree(forget);
 311                         continue;
 312                 }
 313
 314                 sg_init_one(&sg, forget, sizeof(*forget));
 315
 316                 /* Enqueue the request */
 317                 dev_dbg(&vq->vdev->dev, "%s\n", __func__);
 318                 ret = virtqueue_add_sgs(vq, sgs, 1, 0, forget, GFP_ATOMIC);
 319                 if (ret < 0) {
 320                         if (ret == -ENOMEM || ret == -ENOSPC) {
 321                                 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n",
 322                                          ret);
 323                                 list_add_tail(&forget->list,
 324                                                 &fsvq->queued_reqs);
 325                                 schedule_delayed_work(&fsvq->dispatch_work,
 326                                                 msecs_to_jiffies(1));
 327                         } else {
 328                                 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n",
 329                                          ret);
 330                                 kfree(forget);
 331                         }
 332                         spin_unlock(&fsvq->lock);
 333                         return;
 334                 }
 335
 336                 fsvq->in_flight++;
 337                 notify = virtqueue_kick_prepare(vq);
 338                 spin_unlock(&fsvq->lock);
 339
 340                 if (notify)
 341                         virtqueue_notify(vq);
 342                 pr_debug("virtio-fs: worker %s dispatched one forget request.\n",
 343                          __func__);
 344         }
 345 }
 346
 347 /* Allocate and copy args into req->argbuf */
 348 static int copy_args_to_argbuf(struct fuse_req *req)
 349 {
 350         struct fuse_args *args = req->args;
 351         unsigned int offset = 0;
 352         unsigned int num_in;
 353         unsigned int num_out;
 354         unsigned int len;
 355         unsigned int i;
 356
 357         num_in = args->in_numargs - args->in_pages;
 358         num_out = args->out_numargs - args->out_pages;
 359         len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) +
 360               fuse_len_args(num_out, args->out_args);
 361
 362         req->argbuf = kmalloc(len, GFP_ATOMIC);
 363         if (!req->argbuf)
 364                 return -ENOMEM;
 365
 366         for (i = 0; i < num_in; i++) {
 367                 memcpy(req->argbuf + offset,
 368                        args->in_args[i].value,
 369                        args->in_args[i].size);
 370                 offset += args->in_args[i].size;
 371         }
 372
 373         return 0;
 374 }
 375
 376 /* Copy args out of and free req->argbuf */
 377 static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req)
 378 {
 379         unsigned int remaining;
 380         unsigned int offset;
 381         unsigned int num_in;
 382         unsigned int num_out;
 383         unsigned int i;
 384
 385         remaining = req->out.h.len - sizeof(req->out.h);
 386         num_in = args->in_numargs - args->in_pages;
 387         num_out = args->out_numargs - args->out_pages;
 388         offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args);
 389
 390         for (i = 0; i < num_out; i++) {
 391                 unsigned int argsize = args->out_args[i].size;
 392
 393                 if (args->out_argvar &&
 394                     i == args->out_numargs - 1 &&
 395                     argsize > remaining) {
 396                         argsize = remaining;
 397                 }
 398
 399                 memcpy(args->out_args[i].value, req->argbuf + offset, argsize);
 400                 offset += argsize;
 401
 402                 if (i != args->out_numargs - 1)
 403                         remaining -= argsize;
 404         }
 405
 406         /* Store the actual size of the variable-length arg */
 407         if (args->out_argvar)
 408                 args->out_args[args->out_numargs - 1].size = remaining;
 409
 410         kfree(req->argbuf);
 411         req->argbuf = NULL;
 412 }
 413
 414 /* Work function for request completion */
 415 static void virtio_fs_requests_done_work(struct work_struct *work)
 416 {
 417         struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
 418                                                  done_work);
 419         struct fuse_pqueue *fpq = &fsvq->fud->pq;
 420         struct fuse_conn *fc = fsvq->fud->fc;
 421         struct virtqueue *vq = fsvq->vq;
 422         struct fuse_req *req;
 423         struct fuse_args_pages *ap;
 424         struct fuse_req *next;
 425         struct fuse_args *args;
 426         unsigned int len, i, thislen;
 427         struct page *page;
 428         LIST_HEAD(reqs);
 429
 430         /* Collect completed requests off the virtqueue */
 431         spin_lock(&fsvq->lock);
 432         do {
 433                 virtqueue_disable_cb(vq);
 434
 435                 while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
 436                         spin_lock(&fpq->lock);
 437                         list_move_tail(&req->list, &reqs);
 438                         spin_unlock(&fpq->lock);
 439                 }
 440         } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
 441         spin_unlock(&fsvq->lock);
 442
 443         /* End requests */
 444         list_for_each_entry_safe(req, next, &reqs, list) {
 445                 /*
 446                  * TODO verify that server properly follows FUSE protocol
 447                  * (oh.uniq, oh.len)
 448                  */
 449                 args = req->args;
 450                 copy_args_from_argbuf(args, req);
 451
 452                 if (args->out_pages && args->page_zeroing) {
 453                         len = args->out_args[args->out_numargs - 1].size;
 454                         ap = container_of(args, typeof(*ap), args);
 455                         for (i = 0; i < ap->num_pages; i++) {
 456                                 thislen = ap->descs[i].length;
 457                                 if (len < thislen) {
 458                                         WARN_ON(ap->descs[i].offset);
 459                                         page = ap->pages[i];
 460                                         zero_user_segment(page, len, thislen);
 461                                         len = 0;
 462                                 } else {
 463                                         len -= thislen;
 464                                 }
 465                         }
 466                 }
 467
 468                 spin_lock(&fpq->lock);
 469                 clear_bit(FR_SENT, &req->flags);
 470                 list_del_init(&req->list);
 471                 spin_unlock(&fpq->lock);
 472
 473                 fuse_request_end(fc, req);
 474                 spin_lock(&fsvq->lock);
 475                 fsvq->in_flight--;
 476                 spin_unlock(&fsvq->lock);
 477         }
 478 }
 479
 480 /* Virtqueue interrupt handler */
 481 static void virtio_fs_vq_done(struct virtqueue *vq)
 482 {
 483         struct virtio_fs_vq *fsvq = vq_to_fsvq(vq);
 484
 485         dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name);
 486
 487         schedule_work(&fsvq->done_work);
 488 }
 489
 490 /* Initialize virtqueues */
 491 static int virtio_fs_setup_vqs(struct virtio_device *vdev,
 492                                struct virtio_fs *fs)
 493 {
 494         struct virtqueue **vqs;
 495         vq_callback_t **callbacks;
 496         const char **names;
 497         unsigned int i;
 498         int ret = 0;
 499
 500         virtio_cread(vdev, struct virtio_fs_config, num_request_queues,
 501                      &fs->num_request_queues);
 502         if (fs->num_request_queues == 0)
 503                 return -EINVAL;
 504
 505         fs->nvqs = 1 + fs->num_request_queues;
 506         fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL);
 507         if (!fs->vqs)
 508                 return -ENOMEM;
 509
 510         vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL);
 511         callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]),
 512                                         GFP_KERNEL);
 513         names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL);
 514         if (!vqs || !callbacks || !names) {
 515                 ret = -ENOMEM;
 516                 goto out;
 517         }
 518
 519         callbacks[VQ_HIPRIO] = virtio_fs_vq_done;
 520         snprintf(fs->vqs[VQ_HIPRIO].name, sizeof(fs->vqs[VQ_HIPRIO].name),
 521                         "hiprio");
 522         names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name;
 523         INIT_WORK(&fs->vqs[VQ_HIPRIO].done_work, virtio_fs_hiprio_done_work);
 524         INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].queued_reqs);
 525         INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].end_reqs);
 526         INIT_DELAYED_WORK(&fs->vqs[VQ_HIPRIO].dispatch_work,
 527                         virtio_fs_hiprio_dispatch_work);
 528         spin_lock_init(&fs->vqs[VQ_HIPRIO].lock);
 529
 530         /* Initialize the requests virtqueues */
 531         for (i = VQ_REQUEST; i < fs->nvqs; i++) {
 532                 spin_lock_init(&fs->vqs[i].lock);
 533                 INIT_WORK(&fs->vqs[i].done_work, virtio_fs_requests_done_work);
 534                 INIT_DELAYED_WORK(&fs->vqs[i].dispatch_work,
 535                                   virtio_fs_request_dispatch_work);
 536                 INIT_LIST_HEAD(&fs->vqs[i].queued_reqs);
 537                 INIT_LIST_HEAD(&fs->vqs[i].end_reqs);
 538                 snprintf(fs->vqs[i].name, sizeof(fs->vqs[i].name),
 539                          "requests.%u", i - VQ_REQUEST);
 540                 callbacks[i] = virtio_fs_vq_done;
 541                 names[i] = fs->vqs[i].name;
 542         }
 543
 544         ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL);
 545         if (ret < 0)
 546                 goto out;
 547
 548         for (i = 0; i < fs->nvqs; i++)
 549                 fs->vqs[i].vq = vqs[i];
 550
 551         virtio_fs_start_all_queues(fs);
 552 out:
 553         kfree(names);
 554         kfree(callbacks);
 555         kfree(vqs);
 556         if (ret)
 557                 kfree(fs->vqs);
 558         return ret;
 559 }
 560
 561 /* Free virtqueues (device must already be reset) */
 562 static void virtio_fs_cleanup_vqs(struct virtio_device *vdev,
 563                                   struct virtio_fs *fs)
 564 {
 565         vdev->config->del_vqs(vdev);
 566 }
 567
 568 static int virtio_fs_probe(struct virtio_device *vdev)
 569 {
 570         struct virtio_fs *fs;
 571         int ret;
 572
 573         fs = kzalloc(sizeof(*fs), GFP_KERNEL);
 574         if (!fs)
 575                 return -ENOMEM;
 576         kref_init(&fs->refcount);
 577         vdev->priv = fs;
 578
 579         ret = virtio_fs_read_tag(vdev, fs);
 580         if (ret < 0)
 581                 goto out;
 582
 583         ret = virtio_fs_setup_vqs(vdev, fs);
 584         if (ret < 0)
 585                 goto out;
 586
 587         /* TODO vq affinity */
 588
 589         /* Bring the device online in case the filesystem is mounted and
 590          * requests need to be sent before we return.
 591          */
 592         virtio_device_ready(vdev);
 593
 594         ret = virtio_fs_add_instance(fs);
 595         if (ret < 0)
 596                 goto out_vqs;
 597
 598         return 0;
 599
 600 out_vqs:
 601         vdev->config->reset(vdev);
 602         virtio_fs_cleanup_vqs(vdev, fs);
 603
 604 out:
 605         vdev->priv = NULL;
 606         kfree(fs);
 607         return ret;
 608 }
 609
 610 static void virtio_fs_stop_all_queues(struct virtio_fs *fs)
 611 {
 612         struct virtio_fs_vq *fsvq;
 613         int i;
 614
 615         for (i = 0; i < fs->nvqs; i++) {
 616                 fsvq = &fs->vqs[i];
 617                 spin_lock(&fsvq->lock);
 618                 fsvq->connected = false;
 619                 spin_unlock(&fsvq->lock);
 620         }
 621 }
 622
 623 static void virtio_fs_remove(struct virtio_device *vdev)
 624 {
 625         struct virtio_fs *fs = vdev->priv;
 626
 627         mutex_lock(&virtio_fs_mutex);
 628         /* This device is going away. No one should get new reference */
 629         list_del_init(&fs->list);
 630         virtio_fs_stop_all_queues(fs);
 631         virtio_fs_drain_all_queues(fs);
 632         vdev->config->reset(vdev);
 633         virtio_fs_cleanup_vqs(vdev, fs);
 634
 635         vdev->priv = NULL;
 636         /* Put device reference on virtio_fs object */
 637         virtio_fs_put(fs);
 638         mutex_unlock(&virtio_fs_mutex);
 639 }
 640
 641 #ifdef CONFIG_PM_SLEEP
 642 static int virtio_fs_freeze(struct virtio_device *vdev)
 643 {
 644         /* TODO need to save state here */
 645         pr_warn("virtio-fs: suspend/resume not yet supported\n");
 646         return -EOPNOTSUPP;
 647 }
 648
 649 static int virtio_fs_restore(struct virtio_device *vdev)
 650 {
 651          /* TODO need to restore state here */
 652         return 0;
 653 }
 654 #endif /* CONFIG_PM_SLEEP */
 655
 656 const static struct virtio_device_id id_table[] = {
 657         { VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID },
 658         {},
 659 };
 660
 661 const static unsigned int feature_table[] = {};
 662
 663 static struct virtio_driver virtio_fs_driver = {
 664         .driver.name            = KBUILD_MODNAME,
 665         .driver.owner           = THIS_MODULE,
 666         .id_table               = id_table,
 667         .feature_table          = feature_table,
 668         .feature_table_size     = ARRAY_SIZE(feature_table),
 669         .probe                  = virtio_fs_probe,
 670         .remove                 = virtio_fs_remove,
 671 #ifdef CONFIG_PM_SLEEP
 672         .freeze                 = virtio_fs_freeze,
 673         .restore                = virtio_fs_restore,
 674 #endif
 675 };
 676
 677 static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq)
 678 __releases(fiq->lock)
 679 {
 680         struct fuse_forget_link *link;
 681         struct virtio_fs_forget *forget;
 682         struct scatterlist sg;
 683         struct scatterlist *sgs[] = {&sg};
 684         struct virtio_fs *fs;
 685         struct virtqueue *vq;
 686         struct virtio_fs_vq *fsvq;
 687         bool notify;
 688         u64 unique;
 689         int ret;
 690
 691         link = fuse_dequeue_forget(fiq, 1, NULL);
 692         unique = fuse_get_unique(fiq);
 693
 694         fs = fiq->priv;
 695         fsvq = &fs->vqs[VQ_HIPRIO];
 696         spin_unlock(&fiq->lock);
 697
 698         /* Allocate a buffer for the request */
 699         forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL);
 700
 701         forget->ih = (struct fuse_in_header){
 702                 .opcode = FUSE_FORGET,
 703                 .nodeid = link->forget_one.nodeid,
 704                 .unique = unique,
 705                 .len = sizeof(*forget),
 706         };
 707         forget->arg = (struct fuse_forget_in){
 708                 .nlookup = link->forget_one.nlookup,
 709         };
 710
 711         sg_init_one(&sg, forget, sizeof(*forget));
 712
 713         /* Enqueue the request */
 714         spin_lock(&fsvq->lock);
 715
 716         if (!fsvq->connected) {
 717                 kfree(forget);
 718                 spin_unlock(&fsvq->lock);
 719                 goto out;
 720         }
 721
 722         vq = fsvq->vq;
 723         dev_dbg(&vq->vdev->dev, "%s\n", __func__);
 724
 725         ret = virtqueue_add_sgs(vq, sgs, 1, 0, forget, GFP_ATOMIC);
 726         if (ret < 0) {
 727                 if (ret == -ENOMEM || ret == -ENOSPC) {
 728                         pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later.\n",
 729                                  ret);
 730                         list_add_tail(&forget->list, &fsvq->queued_reqs);
 731                         schedule_delayed_work(&fsvq->dispatch_work,
 732                                         msecs_to_jiffies(1));
 733                 } else {
 734                         pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n",
 735                                  ret);
 736                         kfree(forget);
 737                 }
 738                 spin_unlock(&fsvq->lock);
 739                 goto out;
 740         }
 741
 742         fsvq->in_flight++;
 743         notify = virtqueue_kick_prepare(vq);
 744
 745         spin_unlock(&fsvq->lock);
 746
 747         if (notify)
 748                 virtqueue_notify(vq);
 749 out:
 750         kfree(link);
 751 }
 752
 753 static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq)
 754 __releases(fiq->lock)
 755 {
 756         /*
 757          * TODO interrupts.
 758          *
 759          * Normal fs operations on a local filesystems aren't interruptible.
 760          * Exceptions are blocking lock operations; for example fcntl(F_SETLKW)
 761          * with shared lock between host and guest.
 762          */
 763         spin_unlock(&fiq->lock);
 764 }
 765
 766 /* Return the number of scatter-gather list elements required */
 767 static unsigned int sg_count_fuse_req(struct fuse_req *req)
 768 {
 769         struct fuse_args *args = req->args;
 770         struct fuse_args_pages *ap = container_of(args, typeof(*ap), args);
 771         unsigned int total_sgs = 1 /* fuse_in_header */;
 772
 773         if (args->in_numargs - args->in_pages)
 774                 total_sgs += 1;
 775
 776         if (args->in_pages)
 777                 total_sgs += ap->num_pages;
 778
 779         if (!test_bit(FR_ISREPLY, &req->flags))
 780                 return total_sgs;
 781
 782         total_sgs += 1 /* fuse_out_header */;
 783
 784         if (args->out_numargs - args->out_pages)
 785                 total_sgs += 1;
 786
 787         if (args->out_pages)
 788                 total_sgs += ap->num_pages;
 789
 790         return total_sgs;
 791 }
 792
 793 /* Add pages to scatter-gather list and return number of elements used */
 794 static unsigned int sg_init_fuse_pages(struct scatterlist *sg,
 795                                        struct page **pages,
 796                                        struct fuse_page_desc *page_descs,
 797                                        unsigned int num_pages,
 798                                        unsigned int total_len)
 799 {
 800         unsigned int i;
 801         unsigned int this_len;
 802
 803         for (i = 0; i < num_pages && total_len; i++) {
 804                 sg_init_table(&sg[i], 1);
 805                 this_len =  min(page_descs[i].length, total_len);
 806                 sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset);
 807                 total_len -= this_len;
 808         }
 809
 810         return i;
 811 }
 812
 813 /* Add args to scatter-gather list and return number of elements used */
 814 static unsigned int sg_init_fuse_args(struct scatterlist *sg,
 815                                       struct fuse_req *req,
 816                                       struct fuse_arg *args,
 817                                       unsigned int numargs,
 818                                       bool argpages,
 819                                       void *argbuf,
 820                                       unsigned int *len_used)
 821 {
 822         struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args);
 823         unsigned int total_sgs = 0;
 824         unsigned int len;
 825
 826         len = fuse_len_args(numargs - argpages, args);
 827         if (len)
 828                 sg_init_one(&sg[total_sgs++], argbuf, len);
 829
 830         if (argpages)
 831                 total_sgs += sg_init_fuse_pages(&sg[total_sgs],
 832                                                 ap->pages, ap->descs,
 833                                                 ap->num_pages,
 834                                                 args[numargs - 1].size);
 835
 836         if (len_used)
 837                 *len_used = len;
 838
 839         return total_sgs;
 840 }
 841
 842 /* Add a request to a virtqueue and kick the device */
 843 static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
 844                                  struct fuse_req *req)
 845 {
 846         /* requests need at least 4 elements */
 847         struct scatterlist *stack_sgs[6];
 848         struct scatterlist stack_sg[ARRAY_SIZE(stack_sgs)];
 849         struct scatterlist **sgs = stack_sgs;
 850         struct scatterlist *sg = stack_sg;
 851         struct virtqueue *vq;
 852         struct fuse_args *args = req->args;
 853         unsigned int argbuf_used = 0;
 854         unsigned int out_sgs = 0;
 855         unsigned int in_sgs = 0;
 856         unsigned int total_sgs;
 857         unsigned int i;
 858         int ret;
 859         bool notify;
 860
 861         /* Does the sglist fit on the stack? */
 862         total_sgs = sg_count_fuse_req(req);
 863         if (total_sgs > ARRAY_SIZE(stack_sgs)) {
 864                 sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC);
 865                 sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC);
 866                 if (!sgs || !sg) {
 867                         ret = -ENOMEM;
 868                         goto out;
 869                 }
 870         }
 871
 872         /* Use a bounce buffer since stack args cannot be mapped */
 873         ret = copy_args_to_argbuf(req);
 874         if (ret < 0)
 875                 goto out;
 876
 877         /* Request elements */
 878         sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h));
 879         out_sgs += sg_init_fuse_args(&sg[out_sgs], req,
 880                                      (struct fuse_arg *)args->in_args,
 881                                      args->in_numargs, args->in_pages,
 882                                      req->argbuf, &argbuf_used);
 883
 884         /* Reply elements */
 885         if (test_bit(FR_ISREPLY, &req->flags)) {
 886                 sg_init_one(&sg[out_sgs + in_sgs++],
 887                             &req->out.h, sizeof(req->out.h));
 888                 in_sgs += sg_init_fuse_args(&sg[out_sgs + in_sgs], req,
 889                                             args->out_args, args->out_numargs,
 890                                             args->out_pages,
 891                                             req->argbuf + argbuf_used, NULL);
 892         }
 893
 894         WARN_ON(out_sgs + in_sgs != total_sgs);
 895
 896         for (i = 0; i < total_sgs; i++)
 897                 sgs[i] = &sg[i];
 898
 899         spin_lock(&fsvq->lock);
 900
 901         if (!fsvq->connected) {
 902                 spin_unlock(&fsvq->lock);
 903                 ret = -ENOTCONN;
 904                 goto out;
 905         }
 906
 907         vq = fsvq->vq;
 908         ret = virtqueue_add_sgs(vq, sgs, out_sgs, in_sgs, req, GFP_ATOMIC);
 909         if (ret < 0) {
 910                 spin_unlock(&fsvq->lock);
 911                 goto out;
 912         }
 913
 914         fsvq->in_flight++;
 915         notify = virtqueue_kick_prepare(vq);
 916
 917         spin_unlock(&fsvq->lock);
 918
 919         if (notify)
 920                 virtqueue_notify(vq);
 921
 922 out:
 923         if (ret < 0 && req->argbuf) {
 924                 kfree(req->argbuf);
 925                 req->argbuf = NULL;
 926         }
 927         if (sgs != stack_sgs) {
 928                 kfree(sgs);
 929                 kfree(sg);
 930         }
 931
 932         return ret;
 933 }
 934
 935 static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq)
 936 __releases(fiq->lock)
 937 {
 938         unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */
 939         struct virtio_fs *fs;
 940         struct fuse_conn *fc;
 941         struct fuse_req *req;
 942         struct fuse_pqueue *fpq;
 943         struct virtio_fs_vq *fsvq;
 944         int ret;
 945
 946         WARN_ON(list_empty(&fiq->pending));
 947         req = list_last_entry(&fiq->pending, struct fuse_req, list);
 948         clear_bit(FR_PENDING, &req->flags);
 949         list_del_init(&req->list);
 950         WARN_ON(!list_empty(&fiq->pending));
 951         spin_unlock(&fiq->lock);
 952
 953         fs = fiq->priv;
 954         fc = fs->vqs[queue_id].fud->fc;
 955
 956         pr_debug("%s: opcode %u unique %#llx nodeid %#llx in.len %u out.len %u\n",
 957                   __func__, req->in.h.opcode, req->in.h.unique,
 958                  req->in.h.nodeid, req->in.h.len,
 959                  fuse_len_args(req->args->out_numargs, req->args->out_args));
 960
 961         fpq = &fs->vqs[queue_id].fud->pq;
 962         spin_lock(&fpq->lock);
 963         list_add_tail(&req->list, fpq->processing);
 964         spin_unlock(&fpq->lock);
 965         set_bit(FR_SENT, &req->flags);
 966         /* matches barrier in request_wait_answer() */
 967         smp_mb__after_atomic();
 968
 969 retry:
 970         fsvq = &fs->vqs[queue_id];
 971         ret = virtio_fs_enqueue_req(fsvq, req);
 972         if (ret < 0) {
 973                 if (ret == -ENOMEM || ret == -ENOSPC) {
 974                         /* Virtqueue full. Retry submission */
 975                         /* TODO use completion instead of timeout */
 976                         usleep_range(20, 30);
 977                         goto retry;
 978                 }
 979                 req->out.h.error = ret;
 980                 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret);
 981                 spin_lock(&fpq->lock);
 982                 clear_bit(FR_SENT, &req->flags);
 983                 list_del_init(&req->list);
 984                 spin_unlock(&fpq->lock);
 985
 986                 /* Can't end request in submission context. Use a worker */
 987                 spin_lock(&fsvq->lock);
 988                 list_add_tail(&req->list, &fsvq->end_reqs);
 989                 schedule_delayed_work(&fsvq->dispatch_work, 0);
 990                 spin_unlock(&fsvq->lock);
 991                 return;
 992         }
 993 }
 994
 995 const static struct fuse_iqueue_ops virtio_fs_fiq_ops = {
 996         .wake_forget_and_unlock         = virtio_fs_wake_forget_and_unlock,
 997         .wake_interrupt_and_unlock      = virtio_fs_wake_interrupt_and_unlock,
 998         .wake_pending_and_unlock        = virtio_fs_wake_pending_and_unlock,
 999         .release                        = virtio_fs_fiq_release,
1000 };
1001
1002 static int virtio_fs_fill_super(struct super_block *sb)
1003 {
1004         struct fuse_conn *fc = get_fuse_conn_super(sb);
1005         struct virtio_fs *fs = fc->iq.priv;
1006         unsigned int i;
1007         int err;
1008         struct fuse_fs_context ctx = {
1009                 .rootmode = S_IFDIR,
1010                 .default_permissions = 1,
1011                 .allow_other = 1,
1012                 .max_read = UINT_MAX,
1013                 .blksize = 512,
1014                 .destroy = true,
1015                 .no_control = true,
1016                 .no_force_umount = true,
1017                 .no_mount_options = true,
1018         };
1019
1020         mutex_lock(&virtio_fs_mutex);
1021
1022         /* After holding mutex, make sure virtiofs device is still there.
1023          * Though we are holding a reference to it, drive ->remove might
1024          * still have cleaned up virtual queues. In that case bail out.
1025          */
1026         err = -EINVAL;
1027         if (list_empty(&fs->list)) {
1028                 pr_info("virtio-fs: tag <%s> not found\n", fs->tag);
1029                 goto err;
1030         }
1031
1032         err = -ENOMEM;
1033         /* Allocate fuse_dev for hiprio and notification queues */
1034         for (i = 0; i < VQ_REQUEST; i++) {
1035                 struct virtio_fs_vq *fsvq = &fs->vqs[i];
1036
1037                 fsvq->fud = fuse_dev_alloc();
1038                 if (!fsvq->fud)
1039                         goto err_free_fuse_devs;
1040         }
1041
1042         ctx.fudptr = (void **)&fs->vqs[VQ_REQUEST].fud;
1043         err = fuse_fill_super_common(sb, &ctx);
1044         if (err < 0)
1045                 goto err_free_fuse_devs;
1046
1047         fc = fs->vqs[VQ_REQUEST].fud->fc;
1048
1049         for (i = 0; i < fs->nvqs; i++) {
1050                 struct virtio_fs_vq *fsvq = &fs->vqs[i];
1051
1052                 if (i == VQ_REQUEST)
1053                         continue; /* already initialized */
1054                 fuse_dev_install(fsvq->fud, fc);
1055         }
1056
1057         /* Previous unmount will stop all queues. Start these again */
1058         virtio_fs_start_all_queues(fs);
1059         fuse_send_init(fc);
1060         mutex_unlock(&virtio_fs_mutex);
1061         return 0;
1062
1063 err_free_fuse_devs:
1064         virtio_fs_free_devs(fs);
1065 err:
1066         mutex_unlock(&virtio_fs_mutex);
1067         return err;
1068 }
1069
1070 static void virtio_kill_sb(struct super_block *sb)
1071 {
1072         struct fuse_conn *fc = get_fuse_conn_super(sb);
1073         struct virtio_fs *vfs;
1074         struct virtio_fs_vq *fsvq;
1075
1076         /* If mount failed, we can still be called without any fc */
1077         if (!fc)
1078                 return fuse_kill_sb_anon(sb);
1079
1080         vfs = fc->iq.priv;
1081         fsvq = &vfs->vqs[VQ_HIPRIO];
1082
1083         /* Stop forget queue. Soon destroy will be sent */
1084         spin_lock(&fsvq->lock);
1085         fsvq->connected = false;
1086         spin_unlock(&fsvq->lock);
1087         virtio_fs_drain_all_queues(vfs);
1088
1089         fuse_kill_sb_anon(sb);
1090
1091         /* fuse_kill_sb_anon() must have sent destroy. Stop all queues
1092          * and drain one more time and free fuse devices. Freeing fuse
1093          * devices will drop their reference on fuse_conn and that in
1094          * turn will drop its reference on virtio_fs object.
1095          */
1096         virtio_fs_stop_all_queues(vfs);
1097         virtio_fs_drain_all_queues(vfs);
1098         virtio_fs_free_devs(vfs);
1099 }
1100
1101 static int virtio_fs_test_super(struct super_block *sb,
1102                                 struct fs_context *fsc)
1103 {
1104         struct fuse_conn *fc = fsc->s_fs_info;
1105
1106         return fc->iq.priv == get_fuse_conn_super(sb)->iq.priv;
1107 }
1108
1109 static int virtio_fs_set_super(struct super_block *sb,
1110                                struct fs_context *fsc)
1111 {
1112         int err;
1113
1114         err = get_anon_bdev(&sb->s_dev);
1115         if (!err)
1116                 fuse_conn_get(fsc->s_fs_info);
1117
1118         return err;
1119 }
1120
1121 static int virtio_fs_get_tree(struct fs_context *fsc)
1122 {
1123         struct virtio_fs *fs;
1124         struct super_block *sb;
1125         struct fuse_conn *fc;
1126         int err;
1127
1128         /* This gets a reference on virtio_fs object. This ptr gets installed
1129          * in fc->iq->priv. Once fuse_conn is going away, it calls ->put()
1130          * to drop the reference to this object.
1131          */
1132         fs = virtio_fs_find_instance(fsc->source);
1133         if (!fs) {
1134                 pr_info("virtio-fs: tag <%s> not found\n", fsc->source);
1135                 return -EINVAL;
1136         }
1137
1138         fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL);
1139         if (!fc) {
1140                 mutex_lock(&virtio_fs_mutex);
1141                 virtio_fs_put(fs);
1142                 mutex_unlock(&virtio_fs_mutex);
1143                 return -ENOMEM;
1144         }
1145
1146         fuse_conn_init(fc, get_user_ns(current_user_ns()), &virtio_fs_fiq_ops,
1147                        fs);
1148         fc->release = fuse_free_conn;
1149         fc->delete_stale = true;
1150
1151         fsc->s_fs_info = fc;
1152         sb = sget_fc(fsc, virtio_fs_test_super, virtio_fs_set_super);
1153         fuse_conn_put(fc);
1154         if (IS_ERR(sb))
1155                 return PTR_ERR(sb);
1156
1157         if (!sb->s_root) {
1158                 err = virtio_fs_fill_super(sb);
1159                 if (err) {
1160                         deactivate_locked_super(sb);
1161                         return err;
1162                 }
1163
1164                 sb->s_flags |= SB_ACTIVE;
1165         }
1166
1167         WARN_ON(fsc->root);
1168         fsc->root = dget(sb->s_root);
1169         return 0;
1170 }
1171
1172 static const struct fs_context_operations virtio_fs_context_ops = {
1173         .get_tree       = virtio_fs_get_tree,
1174 };
1175
1176 static int virtio_fs_init_fs_context(struct fs_context *fsc)
1177 {
1178         fsc->ops = &virtio_fs_context_ops;
1179         return 0;
1180 }
1181
1182 static struct file_system_type virtio_fs_type = {
1183         .owner          = THIS_MODULE,
1184         .name           = "virtiofs",
1185         .init_fs_context = virtio_fs_init_fs_context,
1186         .kill_sb        = virtio_kill_sb,
1187 };
1188
1189 static int __init virtio_fs_init(void)
1190 {
1191         int ret;
1192
1193         ret = register_virtio_driver(&virtio_fs_driver);
1194         if (ret < 0)
1195                 return ret;
1196
1197         ret = register_filesystem(&virtio_fs_type);
1198         if (ret < 0) {
1199                 unregister_virtio_driver(&virtio_fs_driver);
1200                 return ret;
1201         }
1202
1203         return 0;
1204 }
1205 module_init(virtio_fs_init);
1206
1207 static void __exit virtio_fs_exit(void)
1208 {
1209         unregister_filesystem(&virtio_fs_type);
1210         unregister_virtio_driver(&virtio_fs_driver);
1211 }
1212 module_exit(virtio_fs_exit);
1213
1214 MODULE_AUTHOR("Stefan Hajnoczi <stefanha@redhat.com>");
1215 MODULE_DESCRIPTION("Virtio Filesystem");
1216 MODULE_LICENSE("GPL");
1217 MODULE_ALIAS_FS(KBUILD_MODNAME);
1218 MODULE_DEVICE_TABLE(virtio, id_table);