1 // SPDX-License-Identifier: GPL-2.0
3 * virtio-fs: Virtio Filesystem
4 * Copyright (C) 2018 Red Hat, Inc.
8 #include <linux/module.h>
9 #include <linux/virtio.h>
10 #include <linux/virtio_fs.h>
11 #include <linux/delay.h>
12 #include <linux/fs_context.h>
13 #include <linux/highmem.h>
16 /* List of virtio-fs device instances and a lock for the list. Also provides
17 * mutual exclusion in device removal and mounting path
19 static DEFINE_MUTEX(virtio_fs_mutex);
20 static LIST_HEAD(virtio_fs_instances);
27 /* Per-virtqueue state */
30 struct virtqueue *vq; /* protected by ->lock */
31 struct work_struct done_work;
32 struct list_head queued_reqs;
33 struct list_head end_reqs; /* End these requests */
34 struct delayed_work dispatch_work;
39 } ____cacheline_aligned_in_smp;
41 /* A virtio-fs device instance */
44 struct list_head list; /* on virtio_fs_instances */
46 struct virtio_fs_vq *vqs;
47 unsigned int nvqs; /* number of virtqueues */
48 unsigned int num_request_queues; /* number of request queues */
51 struct virtio_fs_forget {
52 struct fuse_in_header ih;
53 struct fuse_forget_in arg;
54 /* This request can be temporarily queued on virt queue */
55 struct list_head list;
58 static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
59 struct fuse_req *req, bool in_flight);
61 static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq)
63 struct virtio_fs *fs = vq->vdev->priv;
65 return &fs->vqs[vq->index];
68 static inline struct fuse_pqueue *vq_to_fpq(struct virtqueue *vq)
70 return &vq_to_fsvq(vq)->fud->pq;
73 /* Should be called with fsvq->lock held. */
74 static inline void inc_in_flight_req(struct virtio_fs_vq *fsvq)
79 /* Should be called with fsvq->lock held. */
80 static inline void dec_in_flight_req(struct virtio_fs_vq *fsvq)
82 WARN_ON(fsvq->in_flight <= 0);
86 static void release_virtio_fs_obj(struct kref *ref)
88 struct virtio_fs *vfs = container_of(ref, struct virtio_fs, refcount);
94 /* Make sure virtiofs_mutex is held */
95 static void virtio_fs_put(struct virtio_fs *fs)
97 kref_put(&fs->refcount, release_virtio_fs_obj);
100 static void virtio_fs_fiq_release(struct fuse_iqueue *fiq)
102 struct virtio_fs *vfs = fiq->priv;
104 mutex_lock(&virtio_fs_mutex);
106 mutex_unlock(&virtio_fs_mutex);
109 static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq)
111 WARN_ON(fsvq->in_flight < 0);
113 /* Wait for in flight requests to finish.*/
115 spin_lock(&fsvq->lock);
116 if (!fsvq->in_flight) {
117 spin_unlock(&fsvq->lock);
120 spin_unlock(&fsvq->lock);
121 /* TODO use completion instead of timeout */
122 usleep_range(1000, 2000);
125 flush_work(&fsvq->done_work);
126 flush_delayed_work(&fsvq->dispatch_work);
129 static void virtio_fs_drain_all_queues(struct virtio_fs *fs)
131 struct virtio_fs_vq *fsvq;
134 for (i = 0; i < fs->nvqs; i++) {
136 virtio_fs_drain_queue(fsvq);
140 static void virtio_fs_start_all_queues(struct virtio_fs *fs)
142 struct virtio_fs_vq *fsvq;
145 for (i = 0; i < fs->nvqs; i++) {
147 spin_lock(&fsvq->lock);
148 fsvq->connected = true;
149 spin_unlock(&fsvq->lock);
153 /* Add a new instance to the list or return -EEXIST if tag name exists*/
154 static int virtio_fs_add_instance(struct virtio_fs *fs)
156 struct virtio_fs *fs2;
157 bool duplicate = false;
159 mutex_lock(&virtio_fs_mutex);
161 list_for_each_entry(fs2, &virtio_fs_instances, list) {
162 if (strcmp(fs->tag, fs2->tag) == 0)
167 list_add_tail(&fs->list, &virtio_fs_instances);
169 mutex_unlock(&virtio_fs_mutex);
176 /* Return the virtio_fs with a given tag, or NULL */
177 static struct virtio_fs *virtio_fs_find_instance(const char *tag)
179 struct virtio_fs *fs;
181 mutex_lock(&virtio_fs_mutex);
183 list_for_each_entry(fs, &virtio_fs_instances, list) {
184 if (strcmp(fs->tag, tag) == 0) {
185 kref_get(&fs->refcount);
190 fs = NULL; /* not found */
193 mutex_unlock(&virtio_fs_mutex);
198 static void virtio_fs_free_devs(struct virtio_fs *fs)
202 for (i = 0; i < fs->nvqs; i++) {
203 struct virtio_fs_vq *fsvq = &fs->vqs[i];
208 fuse_dev_free(fsvq->fud);
213 /* Read filesystem name from virtio config into fs->tag (must kfree()). */
214 static int virtio_fs_read_tag(struct virtio_device *vdev, struct virtio_fs *fs)
216 char tag_buf[sizeof_field(struct virtio_fs_config, tag)];
220 virtio_cread_bytes(vdev, offsetof(struct virtio_fs_config, tag),
221 &tag_buf, sizeof(tag_buf));
222 end = memchr(tag_buf, '\0', sizeof(tag_buf));
224 return -EINVAL; /* empty tag */
226 end = &tag_buf[sizeof(tag_buf)];
229 fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL);
232 memcpy(fs->tag, tag_buf, len);
237 /* Work function for hiprio completion */
238 static void virtio_fs_hiprio_done_work(struct work_struct *work)
240 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
242 struct virtqueue *vq = fsvq->vq;
244 /* Free completed FUSE_FORGET requests */
245 spin_lock(&fsvq->lock);
250 virtqueue_disable_cb(vq);
252 while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
254 dec_in_flight_req(fsvq);
256 } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
257 spin_unlock(&fsvq->lock);
260 static void virtio_fs_request_dispatch_work(struct work_struct *work)
262 struct fuse_req *req;
263 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
265 struct fuse_conn *fc = fsvq->fud->fc;
268 pr_debug("virtio-fs: worker %s called.\n", __func__);
270 spin_lock(&fsvq->lock);
271 req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req,
274 spin_unlock(&fsvq->lock);
278 list_del_init(&req->list);
279 spin_unlock(&fsvq->lock);
280 fuse_request_end(fc, req);
283 /* Dispatch pending requests */
285 spin_lock(&fsvq->lock);
286 req = list_first_entry_or_null(&fsvq->queued_reqs,
287 struct fuse_req, list);
289 spin_unlock(&fsvq->lock);
292 list_del_init(&req->list);
293 spin_unlock(&fsvq->lock);
295 ret = virtio_fs_enqueue_req(fsvq, req, true);
297 if (ret == -ENOMEM || ret == -ENOSPC) {
298 spin_lock(&fsvq->lock);
299 list_add_tail(&req->list, &fsvq->queued_reqs);
300 schedule_delayed_work(&fsvq->dispatch_work,
301 msecs_to_jiffies(1));
302 spin_unlock(&fsvq->lock);
305 req->out.h.error = ret;
306 spin_lock(&fsvq->lock);
307 dec_in_flight_req(fsvq);
308 spin_unlock(&fsvq->lock);
309 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n",
311 fuse_request_end(fc, req);
316 static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
318 struct virtio_fs_forget *forget;
319 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
321 struct virtqueue *vq = fsvq->vq;
322 struct scatterlist sg;
323 struct scatterlist *sgs[] = {&sg};
327 pr_debug("virtio-fs: worker %s called.\n", __func__);
329 spin_lock(&fsvq->lock);
330 forget = list_first_entry_or_null(&fsvq->queued_reqs,
331 struct virtio_fs_forget, list);
333 spin_unlock(&fsvq->lock);
337 list_del(&forget->list);
338 if (!fsvq->connected) {
339 dec_in_flight_req(fsvq);
340 spin_unlock(&fsvq->lock);
345 sg_init_one(&sg, forget, sizeof(*forget));
347 /* Enqueue the request */
348 dev_dbg(&vq->vdev->dev, "%s\n", __func__);
349 ret = virtqueue_add_sgs(vq, sgs, 1, 0, forget, GFP_ATOMIC);
351 if (ret == -ENOMEM || ret == -ENOSPC) {
352 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n",
354 list_add_tail(&forget->list,
356 schedule_delayed_work(&fsvq->dispatch_work,
357 msecs_to_jiffies(1));
359 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n",
361 dec_in_flight_req(fsvq);
364 spin_unlock(&fsvq->lock);
368 notify = virtqueue_kick_prepare(vq);
369 spin_unlock(&fsvq->lock);
372 virtqueue_notify(vq);
373 pr_debug("virtio-fs: worker %s dispatched one forget request.\n",
378 /* Allocate and copy args into req->argbuf */
379 static int copy_args_to_argbuf(struct fuse_req *req)
381 struct fuse_args *args = req->args;
382 unsigned int offset = 0;
384 unsigned int num_out;
388 num_in = args->in_numargs - args->in_pages;
389 num_out = args->out_numargs - args->out_pages;
390 len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) +
391 fuse_len_args(num_out, args->out_args);
393 req->argbuf = kmalloc(len, GFP_ATOMIC);
397 for (i = 0; i < num_in; i++) {
398 memcpy(req->argbuf + offset,
399 args->in_args[i].value,
400 args->in_args[i].size);
401 offset += args->in_args[i].size;
407 /* Copy args out of and free req->argbuf */
408 static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req)
410 unsigned int remaining;
413 unsigned int num_out;
416 remaining = req->out.h.len - sizeof(req->out.h);
417 num_in = args->in_numargs - args->in_pages;
418 num_out = args->out_numargs - args->out_pages;
419 offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args);
421 for (i = 0; i < num_out; i++) {
422 unsigned int argsize = args->out_args[i].size;
424 if (args->out_argvar &&
425 i == args->out_numargs - 1 &&
426 argsize > remaining) {
430 memcpy(args->out_args[i].value, req->argbuf + offset, argsize);
433 if (i != args->out_numargs - 1)
434 remaining -= argsize;
437 /* Store the actual size of the variable-length arg */
438 if (args->out_argvar)
439 args->out_args[args->out_numargs - 1].size = remaining;
445 /* Work function for request completion */
446 static void virtio_fs_requests_done_work(struct work_struct *work)
448 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
450 struct fuse_pqueue *fpq = &fsvq->fud->pq;
451 struct fuse_conn *fc = fsvq->fud->fc;
452 struct virtqueue *vq = fsvq->vq;
453 struct fuse_req *req;
454 struct fuse_args_pages *ap;
455 struct fuse_req *next;
456 struct fuse_args *args;
457 unsigned int len, i, thislen;
461 /* Collect completed requests off the virtqueue */
462 spin_lock(&fsvq->lock);
464 virtqueue_disable_cb(vq);
466 while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
467 spin_lock(&fpq->lock);
468 list_move_tail(&req->list, &reqs);
469 spin_unlock(&fpq->lock);
471 } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
472 spin_unlock(&fsvq->lock);
475 list_for_each_entry_safe(req, next, &reqs, list) {
477 * TODO verify that server properly follows FUSE protocol
481 copy_args_from_argbuf(args, req);
483 if (args->out_pages && args->page_zeroing) {
484 len = args->out_args[args->out_numargs - 1].size;
485 ap = container_of(args, typeof(*ap), args);
486 for (i = 0; i < ap->num_pages; i++) {
487 thislen = ap->descs[i].length;
489 WARN_ON(ap->descs[i].offset);
491 zero_user_segment(page, len, thislen);
499 spin_lock(&fpq->lock);
500 clear_bit(FR_SENT, &req->flags);
501 list_del_init(&req->list);
502 spin_unlock(&fpq->lock);
504 fuse_request_end(fc, req);
505 spin_lock(&fsvq->lock);
506 dec_in_flight_req(fsvq);
507 spin_unlock(&fsvq->lock);
511 /* Virtqueue interrupt handler */
512 static void virtio_fs_vq_done(struct virtqueue *vq)
514 struct virtio_fs_vq *fsvq = vq_to_fsvq(vq);
516 dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name);
518 schedule_work(&fsvq->done_work);
521 /* Initialize virtqueues */
522 static int virtio_fs_setup_vqs(struct virtio_device *vdev,
523 struct virtio_fs *fs)
525 struct virtqueue **vqs;
526 vq_callback_t **callbacks;
531 virtio_cread(vdev, struct virtio_fs_config, num_request_queues,
532 &fs->num_request_queues);
533 if (fs->num_request_queues == 0)
536 fs->nvqs = 1 + fs->num_request_queues;
537 fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL);
541 vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL);
542 callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]),
544 names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL);
545 if (!vqs || !callbacks || !names) {
550 callbacks[VQ_HIPRIO] = virtio_fs_vq_done;
551 snprintf(fs->vqs[VQ_HIPRIO].name, sizeof(fs->vqs[VQ_HIPRIO].name),
553 names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name;
554 INIT_WORK(&fs->vqs[VQ_HIPRIO].done_work, virtio_fs_hiprio_done_work);
555 INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].queued_reqs);
556 INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].end_reqs);
557 INIT_DELAYED_WORK(&fs->vqs[VQ_HIPRIO].dispatch_work,
558 virtio_fs_hiprio_dispatch_work);
559 spin_lock_init(&fs->vqs[VQ_HIPRIO].lock);
561 /* Initialize the requests virtqueues */
562 for (i = VQ_REQUEST; i < fs->nvqs; i++) {
563 spin_lock_init(&fs->vqs[i].lock);
564 INIT_WORK(&fs->vqs[i].done_work, virtio_fs_requests_done_work);
565 INIT_DELAYED_WORK(&fs->vqs[i].dispatch_work,
566 virtio_fs_request_dispatch_work);
567 INIT_LIST_HEAD(&fs->vqs[i].queued_reqs);
568 INIT_LIST_HEAD(&fs->vqs[i].end_reqs);
569 snprintf(fs->vqs[i].name, sizeof(fs->vqs[i].name),
570 "requests.%u", i - VQ_REQUEST);
571 callbacks[i] = virtio_fs_vq_done;
572 names[i] = fs->vqs[i].name;
575 ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL);
579 for (i = 0; i < fs->nvqs; i++)
580 fs->vqs[i].vq = vqs[i];
582 virtio_fs_start_all_queues(fs);
592 /* Free virtqueues (device must already be reset) */
593 static void virtio_fs_cleanup_vqs(struct virtio_device *vdev,
594 struct virtio_fs *fs)
596 vdev->config->del_vqs(vdev);
599 static int virtio_fs_probe(struct virtio_device *vdev)
601 struct virtio_fs *fs;
604 fs = kzalloc(sizeof(*fs), GFP_KERNEL);
607 kref_init(&fs->refcount);
610 ret = virtio_fs_read_tag(vdev, fs);
614 ret = virtio_fs_setup_vqs(vdev, fs);
618 /* TODO vq affinity */
620 /* Bring the device online in case the filesystem is mounted and
621 * requests need to be sent before we return.
623 virtio_device_ready(vdev);
625 ret = virtio_fs_add_instance(fs);
632 vdev->config->reset(vdev);
633 virtio_fs_cleanup_vqs(vdev, fs);
641 static void virtio_fs_stop_all_queues(struct virtio_fs *fs)
643 struct virtio_fs_vq *fsvq;
646 for (i = 0; i < fs->nvqs; i++) {
648 spin_lock(&fsvq->lock);
649 fsvq->connected = false;
650 spin_unlock(&fsvq->lock);
654 static void virtio_fs_remove(struct virtio_device *vdev)
656 struct virtio_fs *fs = vdev->priv;
658 mutex_lock(&virtio_fs_mutex);
659 /* This device is going away. No one should get new reference */
660 list_del_init(&fs->list);
661 virtio_fs_stop_all_queues(fs);
662 virtio_fs_drain_all_queues(fs);
663 vdev->config->reset(vdev);
664 virtio_fs_cleanup_vqs(vdev, fs);
667 /* Put device reference on virtio_fs object */
669 mutex_unlock(&virtio_fs_mutex);
672 #ifdef CONFIG_PM_SLEEP
673 static int virtio_fs_freeze(struct virtio_device *vdev)
675 /* TODO need to save state here */
676 pr_warn("virtio-fs: suspend/resume not yet supported\n");
680 static int virtio_fs_restore(struct virtio_device *vdev)
682 /* TODO need to restore state here */
685 #endif /* CONFIG_PM_SLEEP */
687 const static struct virtio_device_id id_table[] = {
688 { VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID },
692 const static unsigned int feature_table[] = {};
694 static struct virtio_driver virtio_fs_driver = {
695 .driver.name = KBUILD_MODNAME,
696 .driver.owner = THIS_MODULE,
697 .id_table = id_table,
698 .feature_table = feature_table,
699 .feature_table_size = ARRAY_SIZE(feature_table),
700 .probe = virtio_fs_probe,
701 .remove = virtio_fs_remove,
702 #ifdef CONFIG_PM_SLEEP
703 .freeze = virtio_fs_freeze,
704 .restore = virtio_fs_restore,
708 static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq)
709 __releases(fiq->lock)
711 struct fuse_forget_link *link;
712 struct virtio_fs_forget *forget;
713 struct scatterlist sg;
714 struct scatterlist *sgs[] = {&sg};
715 struct virtio_fs *fs;
716 struct virtqueue *vq;
717 struct virtio_fs_vq *fsvq;
722 link = fuse_dequeue_forget(fiq, 1, NULL);
723 unique = fuse_get_unique(fiq);
726 fsvq = &fs->vqs[VQ_HIPRIO];
727 spin_unlock(&fiq->lock);
729 /* Allocate a buffer for the request */
730 forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL);
732 forget->ih = (struct fuse_in_header){
733 .opcode = FUSE_FORGET,
734 .nodeid = link->forget_one.nodeid,
736 .len = sizeof(*forget),
738 forget->arg = (struct fuse_forget_in){
739 .nlookup = link->forget_one.nlookup,
742 sg_init_one(&sg, forget, sizeof(*forget));
744 /* Enqueue the request */
745 spin_lock(&fsvq->lock);
747 if (!fsvq->connected) {
749 spin_unlock(&fsvq->lock);
754 dev_dbg(&vq->vdev->dev, "%s\n", __func__);
756 ret = virtqueue_add_sgs(vq, sgs, 1, 0, forget, GFP_ATOMIC);
758 if (ret == -ENOMEM || ret == -ENOSPC) {
759 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later.\n",
761 list_add_tail(&forget->list, &fsvq->queued_reqs);
762 schedule_delayed_work(&fsvq->dispatch_work,
763 msecs_to_jiffies(1));
764 inc_in_flight_req(fsvq);
766 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n",
770 spin_unlock(&fsvq->lock);
774 inc_in_flight_req(fsvq);
775 notify = virtqueue_kick_prepare(vq);
777 spin_unlock(&fsvq->lock);
780 virtqueue_notify(vq);
785 static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq)
786 __releases(fiq->lock)
791 * Normal fs operations on a local filesystems aren't interruptible.
792 * Exceptions are blocking lock operations; for example fcntl(F_SETLKW)
793 * with shared lock between host and guest.
795 spin_unlock(&fiq->lock);
798 /* Return the number of scatter-gather list elements required */
799 static unsigned int sg_count_fuse_req(struct fuse_req *req)
801 struct fuse_args *args = req->args;
802 struct fuse_args_pages *ap = container_of(args, typeof(*ap), args);
803 unsigned int total_sgs = 1 /* fuse_in_header */;
805 if (args->in_numargs - args->in_pages)
809 total_sgs += ap->num_pages;
811 if (!test_bit(FR_ISREPLY, &req->flags))
814 total_sgs += 1 /* fuse_out_header */;
816 if (args->out_numargs - args->out_pages)
820 total_sgs += ap->num_pages;
825 /* Add pages to scatter-gather list and return number of elements used */
826 static unsigned int sg_init_fuse_pages(struct scatterlist *sg,
828 struct fuse_page_desc *page_descs,
829 unsigned int num_pages,
830 unsigned int total_len)
833 unsigned int this_len;
835 for (i = 0; i < num_pages && total_len; i++) {
836 sg_init_table(&sg[i], 1);
837 this_len = min(page_descs[i].length, total_len);
838 sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset);
839 total_len -= this_len;
845 /* Add args to scatter-gather list and return number of elements used */
846 static unsigned int sg_init_fuse_args(struct scatterlist *sg,
847 struct fuse_req *req,
848 struct fuse_arg *args,
849 unsigned int numargs,
852 unsigned int *len_used)
854 struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args);
855 unsigned int total_sgs = 0;
858 len = fuse_len_args(numargs - argpages, args);
860 sg_init_one(&sg[total_sgs++], argbuf, len);
863 total_sgs += sg_init_fuse_pages(&sg[total_sgs],
864 ap->pages, ap->descs,
866 args[numargs - 1].size);
874 /* Add a request to a virtqueue and kick the device */
875 static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
876 struct fuse_req *req, bool in_flight)
878 /* requests need at least 4 elements */
879 struct scatterlist *stack_sgs[6];
880 struct scatterlist stack_sg[ARRAY_SIZE(stack_sgs)];
881 struct scatterlist **sgs = stack_sgs;
882 struct scatterlist *sg = stack_sg;
883 struct virtqueue *vq;
884 struct fuse_args *args = req->args;
885 unsigned int argbuf_used = 0;
886 unsigned int out_sgs = 0;
887 unsigned int in_sgs = 0;
888 unsigned int total_sgs;
892 struct fuse_pqueue *fpq;
894 /* Does the sglist fit on the stack? */
895 total_sgs = sg_count_fuse_req(req);
896 if (total_sgs > ARRAY_SIZE(stack_sgs)) {
897 sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC);
898 sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC);
905 /* Use a bounce buffer since stack args cannot be mapped */
906 ret = copy_args_to_argbuf(req);
910 /* Request elements */
911 sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h));
912 out_sgs += sg_init_fuse_args(&sg[out_sgs], req,
913 (struct fuse_arg *)args->in_args,
914 args->in_numargs, args->in_pages,
915 req->argbuf, &argbuf_used);
918 if (test_bit(FR_ISREPLY, &req->flags)) {
919 sg_init_one(&sg[out_sgs + in_sgs++],
920 &req->out.h, sizeof(req->out.h));
921 in_sgs += sg_init_fuse_args(&sg[out_sgs + in_sgs], req,
922 args->out_args, args->out_numargs,
924 req->argbuf + argbuf_used, NULL);
927 WARN_ON(out_sgs + in_sgs != total_sgs);
929 for (i = 0; i < total_sgs; i++)
932 spin_lock(&fsvq->lock);
934 if (!fsvq->connected) {
935 spin_unlock(&fsvq->lock);
941 ret = virtqueue_add_sgs(vq, sgs, out_sgs, in_sgs, req, GFP_ATOMIC);
943 spin_unlock(&fsvq->lock);
947 /* Request successfully sent. */
948 fpq = &fsvq->fud->pq;
949 spin_lock(&fpq->lock);
950 list_add_tail(&req->list, fpq->processing);
951 spin_unlock(&fpq->lock);
952 set_bit(FR_SENT, &req->flags);
953 /* matches barrier in request_wait_answer() */
954 smp_mb__after_atomic();
957 inc_in_flight_req(fsvq);
958 notify = virtqueue_kick_prepare(vq);
960 spin_unlock(&fsvq->lock);
963 virtqueue_notify(vq);
966 if (ret < 0 && req->argbuf) {
970 if (sgs != stack_sgs) {
978 static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq)
979 __releases(fiq->lock)
981 unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */
982 struct virtio_fs *fs;
983 struct fuse_req *req;
984 struct virtio_fs_vq *fsvq;
987 WARN_ON(list_empty(&fiq->pending));
988 req = list_last_entry(&fiq->pending, struct fuse_req, list);
989 clear_bit(FR_PENDING, &req->flags);
990 list_del_init(&req->list);
991 WARN_ON(!list_empty(&fiq->pending));
992 spin_unlock(&fiq->lock);
996 pr_debug("%s: opcode %u unique %#llx nodeid %#llx in.len %u out.len %u\n",
997 __func__, req->in.h.opcode, req->in.h.unique,
998 req->in.h.nodeid, req->in.h.len,
999 fuse_len_args(req->args->out_numargs, req->args->out_args));
1001 fsvq = &fs->vqs[queue_id];
1002 ret = virtio_fs_enqueue_req(fsvq, req, false);
1004 if (ret == -ENOMEM || ret == -ENOSPC) {
1006 * Virtqueue full. Retry submission from worker
1007 * context as we might be holding fc->bg_lock.
1009 spin_lock(&fsvq->lock);
1010 list_add_tail(&req->list, &fsvq->queued_reqs);
1011 inc_in_flight_req(fsvq);
1012 schedule_delayed_work(&fsvq->dispatch_work,
1013 msecs_to_jiffies(1));
1014 spin_unlock(&fsvq->lock);
1017 req->out.h.error = ret;
1018 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret);
1020 /* Can't end request in submission context. Use a worker */
1021 spin_lock(&fsvq->lock);
1022 list_add_tail(&req->list, &fsvq->end_reqs);
1023 schedule_delayed_work(&fsvq->dispatch_work, 0);
1024 spin_unlock(&fsvq->lock);
1029 const static struct fuse_iqueue_ops virtio_fs_fiq_ops = {
1030 .wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock,
1031 .wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock,
1032 .wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock,
1033 .release = virtio_fs_fiq_release,
1036 static int virtio_fs_fill_super(struct super_block *sb)
1038 struct fuse_conn *fc = get_fuse_conn_super(sb);
1039 struct virtio_fs *fs = fc->iq.priv;
1042 struct fuse_fs_context ctx = {
1043 .rootmode = S_IFDIR,
1044 .default_permissions = 1,
1046 .max_read = UINT_MAX,
1050 .no_force_umount = true,
1051 .no_mount_options = true,
1054 mutex_lock(&virtio_fs_mutex);
1056 /* After holding mutex, make sure virtiofs device is still there.
1057 * Though we are holding a reference to it, drive ->remove might
1058 * still have cleaned up virtual queues. In that case bail out.
1061 if (list_empty(&fs->list)) {
1062 pr_info("virtio-fs: tag <%s> not found\n", fs->tag);
1067 /* Allocate fuse_dev for hiprio and notification queues */
1068 for (i = 0; i < VQ_REQUEST; i++) {
1069 struct virtio_fs_vq *fsvq = &fs->vqs[i];
1071 fsvq->fud = fuse_dev_alloc();
1073 goto err_free_fuse_devs;
1076 ctx.fudptr = (void **)&fs->vqs[VQ_REQUEST].fud;
1077 err = fuse_fill_super_common(sb, &ctx);
1079 goto err_free_fuse_devs;
1081 fc = fs->vqs[VQ_REQUEST].fud->fc;
1083 for (i = 0; i < fs->nvqs; i++) {
1084 struct virtio_fs_vq *fsvq = &fs->vqs[i];
1086 if (i == VQ_REQUEST)
1087 continue; /* already initialized */
1088 fuse_dev_install(fsvq->fud, fc);
1091 /* Previous unmount will stop all queues. Start these again */
1092 virtio_fs_start_all_queues(fs);
1094 mutex_unlock(&virtio_fs_mutex);
1098 virtio_fs_free_devs(fs);
1100 mutex_unlock(&virtio_fs_mutex);
1104 static void virtio_kill_sb(struct super_block *sb)
1106 struct fuse_conn *fc = get_fuse_conn_super(sb);
1107 struct virtio_fs *vfs;
1108 struct virtio_fs_vq *fsvq;
1110 /* If mount failed, we can still be called without any fc */
1112 return fuse_kill_sb_anon(sb);
1115 fsvq = &vfs->vqs[VQ_HIPRIO];
1117 /* Stop forget queue. Soon destroy will be sent */
1118 spin_lock(&fsvq->lock);
1119 fsvq->connected = false;
1120 spin_unlock(&fsvq->lock);
1121 virtio_fs_drain_all_queues(vfs);
1123 fuse_kill_sb_anon(sb);
1125 /* fuse_kill_sb_anon() must have sent destroy. Stop all queues
1126 * and drain one more time and free fuse devices. Freeing fuse
1127 * devices will drop their reference on fuse_conn and that in
1128 * turn will drop its reference on virtio_fs object.
1130 virtio_fs_stop_all_queues(vfs);
1131 virtio_fs_drain_all_queues(vfs);
1132 virtio_fs_free_devs(vfs);
1135 static int virtio_fs_test_super(struct super_block *sb,
1136 struct fs_context *fsc)
1138 struct fuse_conn *fc = fsc->s_fs_info;
1140 return fc->iq.priv == get_fuse_conn_super(sb)->iq.priv;
1143 static int virtio_fs_set_super(struct super_block *sb,
1144 struct fs_context *fsc)
1148 err = get_anon_bdev(&sb->s_dev);
1150 fuse_conn_get(fsc->s_fs_info);
1155 static int virtio_fs_get_tree(struct fs_context *fsc)
1157 struct virtio_fs *fs;
1158 struct super_block *sb;
1159 struct fuse_conn *fc;
1162 /* This gets a reference on virtio_fs object. This ptr gets installed
1163 * in fc->iq->priv. Once fuse_conn is going away, it calls ->put()
1164 * to drop the reference to this object.
1166 fs = virtio_fs_find_instance(fsc->source);
1168 pr_info("virtio-fs: tag <%s> not found\n", fsc->source);
1172 fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL);
1174 mutex_lock(&virtio_fs_mutex);
1176 mutex_unlock(&virtio_fs_mutex);
1180 fuse_conn_init(fc, get_user_ns(current_user_ns()), &virtio_fs_fiq_ops,
1182 fc->release = fuse_free_conn;
1183 fc->delete_stale = true;
1185 fsc->s_fs_info = fc;
1186 sb = sget_fc(fsc, virtio_fs_test_super, virtio_fs_set_super);
1192 err = virtio_fs_fill_super(sb);
1194 deactivate_locked_super(sb);
1198 sb->s_flags |= SB_ACTIVE;
1202 fsc->root = dget(sb->s_root);
1206 static const struct fs_context_operations virtio_fs_context_ops = {
1207 .get_tree = virtio_fs_get_tree,
1210 static int virtio_fs_init_fs_context(struct fs_context *fsc)
1212 fsc->ops = &virtio_fs_context_ops;
1216 static struct file_system_type virtio_fs_type = {
1217 .owner = THIS_MODULE,
1219 .init_fs_context = virtio_fs_init_fs_context,
1220 .kill_sb = virtio_kill_sb,
1223 static int __init virtio_fs_init(void)
1227 ret = register_virtio_driver(&virtio_fs_driver);
1231 ret = register_filesystem(&virtio_fs_type);
1233 unregister_virtio_driver(&virtio_fs_driver);
1239 module_init(virtio_fs_init);
1241 static void __exit virtio_fs_exit(void)
1243 unregister_filesystem(&virtio_fs_type);
1244 unregister_virtio_driver(&virtio_fs_driver);
1246 module_exit(virtio_fs_exit);
1248 MODULE_AUTHOR("Stefan Hajnoczi <stefanha@redhat.com>");
1249 MODULE_DESCRIPTION("Virtio Filesystem");
1250 MODULE_LICENSE("GPL");
1251 MODULE_ALIAS_FS(KBUILD_MODNAME);
1252 MODULE_DEVICE_TABLE(virtio, id_table);