1 // SPDX-License-Identifier: GPL-2.0
3 * virtio-fs: Virtio Filesystem
4 * Copyright (C) 2018 Red Hat, Inc.
8 #include <linux/module.h>
9 #include <linux/virtio.h>
10 #include <linux/virtio_fs.h>
11 #include <linux/delay.h>
12 #include <linux/fs_context.h>
13 #include <linux/highmem.h>
16 /* List of virtio-fs device instances and a lock for the list. Also provides
17 * mutual exclusion in device removal and mounting path
19 static DEFINE_MUTEX(virtio_fs_mutex);
20 static LIST_HEAD(virtio_fs_instances);
27 /* Per-virtqueue state */
30 struct virtqueue *vq; /* protected by ->lock */
31 struct work_struct done_work;
32 struct list_head queued_reqs;
33 struct list_head end_reqs; /* End these requests */
34 struct delayed_work dispatch_work;
39 } ____cacheline_aligned_in_smp;
41 /* A virtio-fs device instance */
44 struct list_head list; /* on virtio_fs_instances */
46 struct virtio_fs_vq *vqs;
47 unsigned int nvqs; /* number of virtqueues */
48 unsigned int num_request_queues; /* number of request queues */
51 struct virtio_fs_forget {
52 struct fuse_in_header ih;
53 struct fuse_forget_in arg;
54 /* This request can be temporarily queued on virt queue */
55 struct list_head list;
58 static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq)
60 struct virtio_fs *fs = vq->vdev->priv;
62 return &fs->vqs[vq->index];
65 static inline struct fuse_pqueue *vq_to_fpq(struct virtqueue *vq)
67 return &vq_to_fsvq(vq)->fud->pq;
70 static void release_virtio_fs_obj(struct kref *ref)
72 struct virtio_fs *vfs = container_of(ref, struct virtio_fs, refcount);
78 /* Make sure virtiofs_mutex is held */
79 static void virtio_fs_put(struct virtio_fs *fs)
81 kref_put(&fs->refcount, release_virtio_fs_obj);
84 static void virtio_fs_fiq_release(struct fuse_iqueue *fiq)
86 struct virtio_fs *vfs = fiq->priv;
88 mutex_lock(&virtio_fs_mutex);
90 mutex_unlock(&virtio_fs_mutex);
93 static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq)
95 WARN_ON(fsvq->in_flight < 0);
97 /* Wait for in flight requests to finish.*/
99 spin_lock(&fsvq->lock);
100 if (!fsvq->in_flight) {
101 spin_unlock(&fsvq->lock);
104 spin_unlock(&fsvq->lock);
105 /* TODO use completion instead of timeout */
106 usleep_range(1000, 2000);
109 flush_work(&fsvq->done_work);
110 flush_delayed_work(&fsvq->dispatch_work);
113 static inline void drain_hiprio_queued_reqs(struct virtio_fs_vq *fsvq)
115 struct virtio_fs_forget *forget;
117 spin_lock(&fsvq->lock);
119 forget = list_first_entry_or_null(&fsvq->queued_reqs,
120 struct virtio_fs_forget, list);
123 list_del(&forget->list);
126 spin_unlock(&fsvq->lock);
129 static void virtio_fs_drain_all_queues(struct virtio_fs *fs)
131 struct virtio_fs_vq *fsvq;
134 for (i = 0; i < fs->nvqs; i++) {
137 drain_hiprio_queued_reqs(fsvq);
139 virtio_fs_drain_queue(fsvq);
143 static void virtio_fs_start_all_queues(struct virtio_fs *fs)
145 struct virtio_fs_vq *fsvq;
148 for (i = 0; i < fs->nvqs; i++) {
150 spin_lock(&fsvq->lock);
151 fsvq->connected = true;
152 spin_unlock(&fsvq->lock);
156 /* Add a new instance to the list or return -EEXIST if tag name exists*/
157 static int virtio_fs_add_instance(struct virtio_fs *fs)
159 struct virtio_fs *fs2;
160 bool duplicate = false;
162 mutex_lock(&virtio_fs_mutex);
164 list_for_each_entry(fs2, &virtio_fs_instances, list) {
165 if (strcmp(fs->tag, fs2->tag) == 0)
170 list_add_tail(&fs->list, &virtio_fs_instances);
172 mutex_unlock(&virtio_fs_mutex);
179 /* Return the virtio_fs with a given tag, or NULL */
180 static struct virtio_fs *virtio_fs_find_instance(const char *tag)
182 struct virtio_fs *fs;
184 mutex_lock(&virtio_fs_mutex);
186 list_for_each_entry(fs, &virtio_fs_instances, list) {
187 if (strcmp(fs->tag, tag) == 0) {
188 kref_get(&fs->refcount);
193 fs = NULL; /* not found */
196 mutex_unlock(&virtio_fs_mutex);
201 static void virtio_fs_free_devs(struct virtio_fs *fs)
205 for (i = 0; i < fs->nvqs; i++) {
206 struct virtio_fs_vq *fsvq = &fs->vqs[i];
211 fuse_dev_free(fsvq->fud);
216 /* Read filesystem name from virtio config into fs->tag (must kfree()). */
217 static int virtio_fs_read_tag(struct virtio_device *vdev, struct virtio_fs *fs)
219 char tag_buf[sizeof_field(struct virtio_fs_config, tag)];
223 virtio_cread_bytes(vdev, offsetof(struct virtio_fs_config, tag),
224 &tag_buf, sizeof(tag_buf));
225 end = memchr(tag_buf, '\0', sizeof(tag_buf));
227 return -EINVAL; /* empty tag */
229 end = &tag_buf[sizeof(tag_buf)];
232 fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL);
235 memcpy(fs->tag, tag_buf, len);
240 /* Work function for hiprio completion */
241 static void virtio_fs_hiprio_done_work(struct work_struct *work)
243 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
245 struct virtqueue *vq = fsvq->vq;
247 /* Free completed FUSE_FORGET requests */
248 spin_lock(&fsvq->lock);
253 virtqueue_disable_cb(vq);
255 while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
259 } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
260 spin_unlock(&fsvq->lock);
263 static void virtio_fs_request_dispatch_work(struct work_struct *work)
265 struct fuse_req *req;
266 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
268 struct fuse_conn *fc = fsvq->fud->fc;
270 pr_debug("virtio-fs: worker %s called.\n", __func__);
272 spin_lock(&fsvq->lock);
273 req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req,
276 spin_unlock(&fsvq->lock);
280 list_del_init(&req->list);
281 spin_unlock(&fsvq->lock);
282 fuse_request_end(fc, req);
286 static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
288 struct virtio_fs_forget *forget;
289 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
291 struct virtqueue *vq = fsvq->vq;
292 struct scatterlist sg;
293 struct scatterlist *sgs[] = {&sg};
297 pr_debug("virtio-fs: worker %s called.\n", __func__);
299 spin_lock(&fsvq->lock);
300 forget = list_first_entry_or_null(&fsvq->queued_reqs,
301 struct virtio_fs_forget, list);
303 spin_unlock(&fsvq->lock);
307 list_del(&forget->list);
308 if (!fsvq->connected) {
309 spin_unlock(&fsvq->lock);
314 sg_init_one(&sg, forget, sizeof(*forget));
316 /* Enqueue the request */
317 dev_dbg(&vq->vdev->dev, "%s\n", __func__);
318 ret = virtqueue_add_sgs(vq, sgs, 1, 0, forget, GFP_ATOMIC);
320 if (ret == -ENOMEM || ret == -ENOSPC) {
321 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n",
323 list_add_tail(&forget->list,
325 schedule_delayed_work(&fsvq->dispatch_work,
326 msecs_to_jiffies(1));
328 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n",
332 spin_unlock(&fsvq->lock);
337 notify = virtqueue_kick_prepare(vq);
338 spin_unlock(&fsvq->lock);
341 virtqueue_notify(vq);
342 pr_debug("virtio-fs: worker %s dispatched one forget request.\n",
347 /* Allocate and copy args into req->argbuf */
348 static int copy_args_to_argbuf(struct fuse_req *req)
350 struct fuse_args *args = req->args;
351 unsigned int offset = 0;
353 unsigned int num_out;
357 num_in = args->in_numargs - args->in_pages;
358 num_out = args->out_numargs - args->out_pages;
359 len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) +
360 fuse_len_args(num_out, args->out_args);
362 req->argbuf = kmalloc(len, GFP_ATOMIC);
366 for (i = 0; i < num_in; i++) {
367 memcpy(req->argbuf + offset,
368 args->in_args[i].value,
369 args->in_args[i].size);
370 offset += args->in_args[i].size;
376 /* Copy args out of and free req->argbuf */
377 static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req)
379 unsigned int remaining;
382 unsigned int num_out;
385 remaining = req->out.h.len - sizeof(req->out.h);
386 num_in = args->in_numargs - args->in_pages;
387 num_out = args->out_numargs - args->out_pages;
388 offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args);
390 for (i = 0; i < num_out; i++) {
391 unsigned int argsize = args->out_args[i].size;
393 if (args->out_argvar &&
394 i == args->out_numargs - 1 &&
395 argsize > remaining) {
399 memcpy(args->out_args[i].value, req->argbuf + offset, argsize);
402 if (i != args->out_numargs - 1)
403 remaining -= argsize;
406 /* Store the actual size of the variable-length arg */
407 if (args->out_argvar)
408 args->out_args[args->out_numargs - 1].size = remaining;
414 /* Work function for request completion */
415 static void virtio_fs_requests_done_work(struct work_struct *work)
417 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
419 struct fuse_pqueue *fpq = &fsvq->fud->pq;
420 struct fuse_conn *fc = fsvq->fud->fc;
421 struct virtqueue *vq = fsvq->vq;
422 struct fuse_req *req;
423 struct fuse_args_pages *ap;
424 struct fuse_req *next;
425 struct fuse_args *args;
426 unsigned int len, i, thislen;
430 /* Collect completed requests off the virtqueue */
431 spin_lock(&fsvq->lock);
433 virtqueue_disable_cb(vq);
435 while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
436 spin_lock(&fpq->lock);
437 list_move_tail(&req->list, &reqs);
438 spin_unlock(&fpq->lock);
440 } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
441 spin_unlock(&fsvq->lock);
444 list_for_each_entry_safe(req, next, &reqs, list) {
446 * TODO verify that server properly follows FUSE protocol
450 copy_args_from_argbuf(args, req);
452 if (args->out_pages && args->page_zeroing) {
453 len = args->out_args[args->out_numargs - 1].size;
454 ap = container_of(args, typeof(*ap), args);
455 for (i = 0; i < ap->num_pages; i++) {
456 thislen = ap->descs[i].length;
458 WARN_ON(ap->descs[i].offset);
460 zero_user_segment(page, len, thislen);
468 spin_lock(&fpq->lock);
469 clear_bit(FR_SENT, &req->flags);
470 list_del_init(&req->list);
471 spin_unlock(&fpq->lock);
473 fuse_request_end(fc, req);
474 spin_lock(&fsvq->lock);
476 spin_unlock(&fsvq->lock);
480 /* Virtqueue interrupt handler */
481 static void virtio_fs_vq_done(struct virtqueue *vq)
483 struct virtio_fs_vq *fsvq = vq_to_fsvq(vq);
485 dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name);
487 schedule_work(&fsvq->done_work);
490 /* Initialize virtqueues */
491 static int virtio_fs_setup_vqs(struct virtio_device *vdev,
492 struct virtio_fs *fs)
494 struct virtqueue **vqs;
495 vq_callback_t **callbacks;
500 virtio_cread(vdev, struct virtio_fs_config, num_request_queues,
501 &fs->num_request_queues);
502 if (fs->num_request_queues == 0)
505 fs->nvqs = 1 + fs->num_request_queues;
506 fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL);
510 vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL);
511 callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]),
513 names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL);
514 if (!vqs || !callbacks || !names) {
519 callbacks[VQ_HIPRIO] = virtio_fs_vq_done;
520 snprintf(fs->vqs[VQ_HIPRIO].name, sizeof(fs->vqs[VQ_HIPRIO].name),
522 names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name;
523 INIT_WORK(&fs->vqs[VQ_HIPRIO].done_work, virtio_fs_hiprio_done_work);
524 INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].queued_reqs);
525 INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].end_reqs);
526 INIT_DELAYED_WORK(&fs->vqs[VQ_HIPRIO].dispatch_work,
527 virtio_fs_hiprio_dispatch_work);
528 spin_lock_init(&fs->vqs[VQ_HIPRIO].lock);
530 /* Initialize the requests virtqueues */
531 for (i = VQ_REQUEST; i < fs->nvqs; i++) {
532 spin_lock_init(&fs->vqs[i].lock);
533 INIT_WORK(&fs->vqs[i].done_work, virtio_fs_requests_done_work);
534 INIT_DELAYED_WORK(&fs->vqs[i].dispatch_work,
535 virtio_fs_request_dispatch_work);
536 INIT_LIST_HEAD(&fs->vqs[i].queued_reqs);
537 INIT_LIST_HEAD(&fs->vqs[i].end_reqs);
538 snprintf(fs->vqs[i].name, sizeof(fs->vqs[i].name),
539 "requests.%u", i - VQ_REQUEST);
540 callbacks[i] = virtio_fs_vq_done;
541 names[i] = fs->vqs[i].name;
544 ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL);
548 for (i = 0; i < fs->nvqs; i++)
549 fs->vqs[i].vq = vqs[i];
551 virtio_fs_start_all_queues(fs);
561 /* Free virtqueues (device must already be reset) */
562 static void virtio_fs_cleanup_vqs(struct virtio_device *vdev,
563 struct virtio_fs *fs)
565 vdev->config->del_vqs(vdev);
568 static int virtio_fs_probe(struct virtio_device *vdev)
570 struct virtio_fs *fs;
573 fs = kzalloc(sizeof(*fs), GFP_KERNEL);
576 kref_init(&fs->refcount);
579 ret = virtio_fs_read_tag(vdev, fs);
583 ret = virtio_fs_setup_vqs(vdev, fs);
587 /* TODO vq affinity */
589 /* Bring the device online in case the filesystem is mounted and
590 * requests need to be sent before we return.
592 virtio_device_ready(vdev);
594 ret = virtio_fs_add_instance(fs);
601 vdev->config->reset(vdev);
602 virtio_fs_cleanup_vqs(vdev, fs);
610 static void virtio_fs_stop_all_queues(struct virtio_fs *fs)
612 struct virtio_fs_vq *fsvq;
615 for (i = 0; i < fs->nvqs; i++) {
617 spin_lock(&fsvq->lock);
618 fsvq->connected = false;
619 spin_unlock(&fsvq->lock);
623 static void virtio_fs_remove(struct virtio_device *vdev)
625 struct virtio_fs *fs = vdev->priv;
627 mutex_lock(&virtio_fs_mutex);
628 /* This device is going away. No one should get new reference */
629 list_del_init(&fs->list);
630 virtio_fs_stop_all_queues(fs);
631 virtio_fs_drain_all_queues(fs);
632 vdev->config->reset(vdev);
633 virtio_fs_cleanup_vqs(vdev, fs);
636 /* Put device reference on virtio_fs object */
638 mutex_unlock(&virtio_fs_mutex);
641 #ifdef CONFIG_PM_SLEEP
642 static int virtio_fs_freeze(struct virtio_device *vdev)
644 /* TODO need to save state here */
645 pr_warn("virtio-fs: suspend/resume not yet supported\n");
649 static int virtio_fs_restore(struct virtio_device *vdev)
651 /* TODO need to restore state here */
654 #endif /* CONFIG_PM_SLEEP */
656 const static struct virtio_device_id id_table[] = {
657 { VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID },
661 const static unsigned int feature_table[] = {};
663 static struct virtio_driver virtio_fs_driver = {
664 .driver.name = KBUILD_MODNAME,
665 .driver.owner = THIS_MODULE,
666 .id_table = id_table,
667 .feature_table = feature_table,
668 .feature_table_size = ARRAY_SIZE(feature_table),
669 .probe = virtio_fs_probe,
670 .remove = virtio_fs_remove,
671 #ifdef CONFIG_PM_SLEEP
672 .freeze = virtio_fs_freeze,
673 .restore = virtio_fs_restore,
677 static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq)
678 __releases(fiq->lock)
680 struct fuse_forget_link *link;
681 struct virtio_fs_forget *forget;
682 struct scatterlist sg;
683 struct scatterlist *sgs[] = {&sg};
684 struct virtio_fs *fs;
685 struct virtqueue *vq;
686 struct virtio_fs_vq *fsvq;
691 link = fuse_dequeue_forget(fiq, 1, NULL);
692 unique = fuse_get_unique(fiq);
695 fsvq = &fs->vqs[VQ_HIPRIO];
696 spin_unlock(&fiq->lock);
698 /* Allocate a buffer for the request */
699 forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL);
701 forget->ih = (struct fuse_in_header){
702 .opcode = FUSE_FORGET,
703 .nodeid = link->forget_one.nodeid,
705 .len = sizeof(*forget),
707 forget->arg = (struct fuse_forget_in){
708 .nlookup = link->forget_one.nlookup,
711 sg_init_one(&sg, forget, sizeof(*forget));
713 /* Enqueue the request */
714 spin_lock(&fsvq->lock);
716 if (!fsvq->connected) {
718 spin_unlock(&fsvq->lock);
723 dev_dbg(&vq->vdev->dev, "%s\n", __func__);
725 ret = virtqueue_add_sgs(vq, sgs, 1, 0, forget, GFP_ATOMIC);
727 if (ret == -ENOMEM || ret == -ENOSPC) {
728 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later.\n",
730 list_add_tail(&forget->list, &fsvq->queued_reqs);
731 schedule_delayed_work(&fsvq->dispatch_work,
732 msecs_to_jiffies(1));
734 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n",
738 spin_unlock(&fsvq->lock);
743 notify = virtqueue_kick_prepare(vq);
745 spin_unlock(&fsvq->lock);
748 virtqueue_notify(vq);
753 static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq)
754 __releases(fiq->lock)
759 * Normal fs operations on a local filesystems aren't interruptible.
760 * Exceptions are blocking lock operations; for example fcntl(F_SETLKW)
761 * with shared lock between host and guest.
763 spin_unlock(&fiq->lock);
766 /* Return the number of scatter-gather list elements required */
767 static unsigned int sg_count_fuse_req(struct fuse_req *req)
769 struct fuse_args *args = req->args;
770 struct fuse_args_pages *ap = container_of(args, typeof(*ap), args);
771 unsigned int total_sgs = 1 /* fuse_in_header */;
773 if (args->in_numargs - args->in_pages)
777 total_sgs += ap->num_pages;
779 if (!test_bit(FR_ISREPLY, &req->flags))
782 total_sgs += 1 /* fuse_out_header */;
784 if (args->out_numargs - args->out_pages)
788 total_sgs += ap->num_pages;
793 /* Add pages to scatter-gather list and return number of elements used */
794 static unsigned int sg_init_fuse_pages(struct scatterlist *sg,
796 struct fuse_page_desc *page_descs,
797 unsigned int num_pages,
798 unsigned int total_len)
801 unsigned int this_len;
803 for (i = 0; i < num_pages && total_len; i++) {
804 sg_init_table(&sg[i], 1);
805 this_len = min(page_descs[i].length, total_len);
806 sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset);
807 total_len -= this_len;
813 /* Add args to scatter-gather list and return number of elements used */
814 static unsigned int sg_init_fuse_args(struct scatterlist *sg,
815 struct fuse_req *req,
816 struct fuse_arg *args,
817 unsigned int numargs,
820 unsigned int *len_used)
822 struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args);
823 unsigned int total_sgs = 0;
826 len = fuse_len_args(numargs - argpages, args);
828 sg_init_one(&sg[total_sgs++], argbuf, len);
831 total_sgs += sg_init_fuse_pages(&sg[total_sgs],
832 ap->pages, ap->descs,
834 args[numargs - 1].size);
842 /* Add a request to a virtqueue and kick the device */
843 static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
844 struct fuse_req *req)
846 /* requests need at least 4 elements */
847 struct scatterlist *stack_sgs[6];
848 struct scatterlist stack_sg[ARRAY_SIZE(stack_sgs)];
849 struct scatterlist **sgs = stack_sgs;
850 struct scatterlist *sg = stack_sg;
851 struct virtqueue *vq;
852 struct fuse_args *args = req->args;
853 unsigned int argbuf_used = 0;
854 unsigned int out_sgs = 0;
855 unsigned int in_sgs = 0;
856 unsigned int total_sgs;
860 struct fuse_pqueue *fpq;
862 /* Does the sglist fit on the stack? */
863 total_sgs = sg_count_fuse_req(req);
864 if (total_sgs > ARRAY_SIZE(stack_sgs)) {
865 sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC);
866 sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC);
873 /* Use a bounce buffer since stack args cannot be mapped */
874 ret = copy_args_to_argbuf(req);
878 /* Request elements */
879 sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h));
880 out_sgs += sg_init_fuse_args(&sg[out_sgs], req,
881 (struct fuse_arg *)args->in_args,
882 args->in_numargs, args->in_pages,
883 req->argbuf, &argbuf_used);
886 if (test_bit(FR_ISREPLY, &req->flags)) {
887 sg_init_one(&sg[out_sgs + in_sgs++],
888 &req->out.h, sizeof(req->out.h));
889 in_sgs += sg_init_fuse_args(&sg[out_sgs + in_sgs], req,
890 args->out_args, args->out_numargs,
892 req->argbuf + argbuf_used, NULL);
895 WARN_ON(out_sgs + in_sgs != total_sgs);
897 for (i = 0; i < total_sgs; i++)
900 spin_lock(&fsvq->lock);
902 if (!fsvq->connected) {
903 spin_unlock(&fsvq->lock);
909 ret = virtqueue_add_sgs(vq, sgs, out_sgs, in_sgs, req, GFP_ATOMIC);
911 spin_unlock(&fsvq->lock);
915 /* Request successfully sent. */
916 fpq = &fsvq->fud->pq;
917 spin_lock(&fpq->lock);
918 list_add_tail(&req->list, fpq->processing);
919 spin_unlock(&fpq->lock);
920 set_bit(FR_SENT, &req->flags);
921 /* matches barrier in request_wait_answer() */
922 smp_mb__after_atomic();
925 notify = virtqueue_kick_prepare(vq);
927 spin_unlock(&fsvq->lock);
930 virtqueue_notify(vq);
933 if (ret < 0 && req->argbuf) {
937 if (sgs != stack_sgs) {
945 static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq)
946 __releases(fiq->lock)
948 unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */
949 struct virtio_fs *fs;
950 struct fuse_conn *fc;
951 struct fuse_req *req;
952 struct virtio_fs_vq *fsvq;
955 WARN_ON(list_empty(&fiq->pending));
956 req = list_last_entry(&fiq->pending, struct fuse_req, list);
957 clear_bit(FR_PENDING, &req->flags);
958 list_del_init(&req->list);
959 WARN_ON(!list_empty(&fiq->pending));
960 spin_unlock(&fiq->lock);
963 fc = fs->vqs[queue_id].fud->fc;
965 pr_debug("%s: opcode %u unique %#llx nodeid %#llx in.len %u out.len %u\n",
966 __func__, req->in.h.opcode, req->in.h.unique,
967 req->in.h.nodeid, req->in.h.len,
968 fuse_len_args(req->args->out_numargs, req->args->out_args));
971 fsvq = &fs->vqs[queue_id];
972 ret = virtio_fs_enqueue_req(fsvq, req);
974 if (ret == -ENOMEM || ret == -ENOSPC) {
975 /* Virtqueue full. Retry submission */
976 /* TODO use completion instead of timeout */
977 usleep_range(20, 30);
980 req->out.h.error = ret;
981 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret);
983 /* Can't end request in submission context. Use a worker */
984 spin_lock(&fsvq->lock);
985 list_add_tail(&req->list, &fsvq->end_reqs);
986 schedule_delayed_work(&fsvq->dispatch_work, 0);
987 spin_unlock(&fsvq->lock);
992 const static struct fuse_iqueue_ops virtio_fs_fiq_ops = {
993 .wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock,
994 .wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock,
995 .wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock,
996 .release = virtio_fs_fiq_release,
999 static int virtio_fs_fill_super(struct super_block *sb)
1001 struct fuse_conn *fc = get_fuse_conn_super(sb);
1002 struct virtio_fs *fs = fc->iq.priv;
1005 struct fuse_fs_context ctx = {
1006 .rootmode = S_IFDIR,
1007 .default_permissions = 1,
1009 .max_read = UINT_MAX,
1013 .no_force_umount = true,
1014 .no_mount_options = true,
1017 mutex_lock(&virtio_fs_mutex);
1019 /* After holding mutex, make sure virtiofs device is still there.
1020 * Though we are holding a reference to it, drive ->remove might
1021 * still have cleaned up virtual queues. In that case bail out.
1024 if (list_empty(&fs->list)) {
1025 pr_info("virtio-fs: tag <%s> not found\n", fs->tag);
1030 /* Allocate fuse_dev for hiprio and notification queues */
1031 for (i = 0; i < VQ_REQUEST; i++) {
1032 struct virtio_fs_vq *fsvq = &fs->vqs[i];
1034 fsvq->fud = fuse_dev_alloc();
1036 goto err_free_fuse_devs;
1039 ctx.fudptr = (void **)&fs->vqs[VQ_REQUEST].fud;
1040 err = fuse_fill_super_common(sb, &ctx);
1042 goto err_free_fuse_devs;
1044 fc = fs->vqs[VQ_REQUEST].fud->fc;
1046 for (i = 0; i < fs->nvqs; i++) {
1047 struct virtio_fs_vq *fsvq = &fs->vqs[i];
1049 if (i == VQ_REQUEST)
1050 continue; /* already initialized */
1051 fuse_dev_install(fsvq->fud, fc);
1054 /* Previous unmount will stop all queues. Start these again */
1055 virtio_fs_start_all_queues(fs);
1057 mutex_unlock(&virtio_fs_mutex);
1061 virtio_fs_free_devs(fs);
1063 mutex_unlock(&virtio_fs_mutex);
1067 static void virtio_kill_sb(struct super_block *sb)
1069 struct fuse_conn *fc = get_fuse_conn_super(sb);
1070 struct virtio_fs *vfs;
1071 struct virtio_fs_vq *fsvq;
1073 /* If mount failed, we can still be called without any fc */
1075 return fuse_kill_sb_anon(sb);
1078 fsvq = &vfs->vqs[VQ_HIPRIO];
1080 /* Stop forget queue. Soon destroy will be sent */
1081 spin_lock(&fsvq->lock);
1082 fsvq->connected = false;
1083 spin_unlock(&fsvq->lock);
1084 virtio_fs_drain_all_queues(vfs);
1086 fuse_kill_sb_anon(sb);
1088 /* fuse_kill_sb_anon() must have sent destroy. Stop all queues
1089 * and drain one more time and free fuse devices. Freeing fuse
1090 * devices will drop their reference on fuse_conn and that in
1091 * turn will drop its reference on virtio_fs object.
1093 virtio_fs_stop_all_queues(vfs);
1094 virtio_fs_drain_all_queues(vfs);
1095 virtio_fs_free_devs(vfs);
1098 static int virtio_fs_test_super(struct super_block *sb,
1099 struct fs_context *fsc)
1101 struct fuse_conn *fc = fsc->s_fs_info;
1103 return fc->iq.priv == get_fuse_conn_super(sb)->iq.priv;
1106 static int virtio_fs_set_super(struct super_block *sb,
1107 struct fs_context *fsc)
1111 err = get_anon_bdev(&sb->s_dev);
1113 fuse_conn_get(fsc->s_fs_info);
1118 static int virtio_fs_get_tree(struct fs_context *fsc)
1120 struct virtio_fs *fs;
1121 struct super_block *sb;
1122 struct fuse_conn *fc;
1125 /* This gets a reference on virtio_fs object. This ptr gets installed
1126 * in fc->iq->priv. Once fuse_conn is going away, it calls ->put()
1127 * to drop the reference to this object.
1129 fs = virtio_fs_find_instance(fsc->source);
1131 pr_info("virtio-fs: tag <%s> not found\n", fsc->source);
1135 fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL);
1137 mutex_lock(&virtio_fs_mutex);
1139 mutex_unlock(&virtio_fs_mutex);
1143 fuse_conn_init(fc, get_user_ns(current_user_ns()), &virtio_fs_fiq_ops,
1145 fc->release = fuse_free_conn;
1146 fc->delete_stale = true;
1148 fsc->s_fs_info = fc;
1149 sb = sget_fc(fsc, virtio_fs_test_super, virtio_fs_set_super);
1155 err = virtio_fs_fill_super(sb);
1157 deactivate_locked_super(sb);
1161 sb->s_flags |= SB_ACTIVE;
1165 fsc->root = dget(sb->s_root);
1169 static const struct fs_context_operations virtio_fs_context_ops = {
1170 .get_tree = virtio_fs_get_tree,
1173 static int virtio_fs_init_fs_context(struct fs_context *fsc)
1175 fsc->ops = &virtio_fs_context_ops;
1179 static struct file_system_type virtio_fs_type = {
1180 .owner = THIS_MODULE,
1182 .init_fs_context = virtio_fs_init_fs_context,
1183 .kill_sb = virtio_kill_sb,
1186 static int __init virtio_fs_init(void)
1190 ret = register_virtio_driver(&virtio_fs_driver);
1194 ret = register_filesystem(&virtio_fs_type);
1196 unregister_virtio_driver(&virtio_fs_driver);
1202 module_init(virtio_fs_init);
1204 static void __exit virtio_fs_exit(void)
1206 unregister_filesystem(&virtio_fs_type);
1207 unregister_virtio_driver(&virtio_fs_driver);
1209 module_exit(virtio_fs_exit);
1211 MODULE_AUTHOR("Stefan Hajnoczi <stefanha@redhat.com>");
1212 MODULE_DESCRIPTION("Virtio Filesystem");
1213 MODULE_LICENSE("GPL");
1214 MODULE_ALIAS_FS(KBUILD_MODNAME);
1215 MODULE_DEVICE_TABLE(virtio, id_table);