1 // SPDX-License-Identifier: GPL-2.0-only
2 /* The industrial I/O core
4 * Copyright (c) 2008 Jonathan Cameron
6 * Handling of buffer allocation / resizing.
8 * Things to look at here.
9 * - Better memory allocation techniques?
10 * - Alternative access techniques?
12 #include <linux/kernel.h>
13 #include <linux/export.h>
14 #include <linux/device.h>
16 #include <linux/cdev.h>
17 #include <linux/slab.h>
18 #include <linux/poll.h>
19 #include <linux/sched/signal.h>
21 #include <linux/iio/iio.h>
23 #include <linux/iio/sysfs.h>
24 #include <linux/iio/buffer.h>
25 #include <linux/iio/buffer_impl.h>
27 static const char * const iio_endian_prefix[] = {
32 static bool iio_buffer_is_active(struct iio_buffer *buf)
34 return !list_empty(&buf->buffer_list);
37 static size_t iio_buffer_data_available(struct iio_buffer *buf)
39 return buf->access->data_available(buf);
42 static int iio_buffer_flush_hwfifo(struct iio_dev *indio_dev,
43 struct iio_buffer *buf, size_t required)
45 if (!indio_dev->info->hwfifo_flush_to_buffer)
48 return indio_dev->info->hwfifo_flush_to_buffer(indio_dev, required);
51 static bool iio_buffer_ready(struct iio_dev *indio_dev, struct iio_buffer *buf,
52 size_t to_wait, int to_flush)
57 /* wakeup if the device was unregistered */
61 /* drain the buffer if it was disabled */
62 if (!iio_buffer_is_active(buf)) {
63 to_wait = min_t(size_t, to_wait, 1);
67 avail = iio_buffer_data_available(buf);
69 if (avail >= to_wait) {
70 /* force a flush for non-blocking reads */
71 if (!to_wait && avail < to_flush)
72 iio_buffer_flush_hwfifo(indio_dev, buf,
78 flushed = iio_buffer_flush_hwfifo(indio_dev, buf,
83 if (avail + flushed >= to_wait)
90 * iio_buffer_read_first_n_outer() - chrdev read for buffer access
91 * @filp: File structure pointer for the char device
92 * @buf: Destination buffer for iio buffer read
93 * @n: First n bytes to read
94 * @f_ps: Long offset provided by the user as a seek position
96 * This function relies on all buffer implementations having an
97 * iio_buffer as their first element.
99 * Return: negative values corresponding to error codes or ret != 0
100 * for ending the reading activity
102 ssize_t iio_buffer_read_first_n_outer(struct file *filp, char __user *buf,
103 size_t n, loff_t *f_ps)
105 struct iio_dev *indio_dev = filp->private_data;
106 struct iio_buffer *rb = indio_dev->buffer;
107 DEFINE_WAIT_FUNC(wait, woken_wake_function);
112 if (!indio_dev->info)
115 if (!rb || !rb->access->read_first_n)
118 datum_size = rb->bytes_per_datum;
121 * If datum_size is 0 there will never be anything to read from the
122 * buffer, so signal end of file now.
127 if (filp->f_flags & O_NONBLOCK)
130 to_wait = min_t(size_t, n / datum_size, rb->watermark);
132 add_wait_queue(&rb->pollq, &wait);
134 if (!indio_dev->info) {
139 if (!iio_buffer_ready(indio_dev, rb, to_wait, n / datum_size)) {
140 if (signal_pending(current)) {
145 wait_woken(&wait, TASK_INTERRUPTIBLE,
146 MAX_SCHEDULE_TIMEOUT);
150 ret = rb->access->read_first_n(rb, n, buf);
151 if (ret == 0 && (filp->f_flags & O_NONBLOCK))
154 remove_wait_queue(&rb->pollq, &wait);
160 * iio_buffer_poll() - poll the buffer to find out if it has data
161 * @filp: File structure pointer for device access
162 * @wait: Poll table structure pointer for which the driver adds
165 * Return: (EPOLLIN | EPOLLRDNORM) if data is available for reading
166 * or 0 for other cases
168 __poll_t iio_buffer_poll(struct file *filp,
169 struct poll_table_struct *wait)
171 struct iio_dev *indio_dev = filp->private_data;
172 struct iio_buffer *rb = indio_dev->buffer;
174 if (!indio_dev->info || rb == NULL)
177 poll_wait(filp, &rb->pollq, wait);
178 if (iio_buffer_ready(indio_dev, rb, rb->watermark, 0))
179 return EPOLLIN | EPOLLRDNORM;
184 * iio_buffer_wakeup_poll - Wakes up the buffer waitqueue
185 * @indio_dev: The IIO device
187 * Wakes up the event waitqueue used for poll(). Should usually
188 * be called when the device is unregistered.
190 void iio_buffer_wakeup_poll(struct iio_dev *indio_dev)
192 if (!indio_dev->buffer)
195 wake_up(&indio_dev->buffer->pollq);
198 void iio_buffer_init(struct iio_buffer *buffer)
200 INIT_LIST_HEAD(&buffer->demux_list);
201 INIT_LIST_HEAD(&buffer->buffer_list);
202 init_waitqueue_head(&buffer->pollq);
203 kref_init(&buffer->ref);
204 if (!buffer->watermark)
205 buffer->watermark = 1;
207 EXPORT_SYMBOL(iio_buffer_init);
210 * iio_buffer_set_attrs - Set buffer specific attributes
211 * @buffer: The buffer for which we are setting attributes
212 * @attrs: Pointer to a null terminated list of pointers to attributes
214 void iio_buffer_set_attrs(struct iio_buffer *buffer,
215 const struct attribute **attrs)
217 buffer->attrs = attrs;
219 EXPORT_SYMBOL_GPL(iio_buffer_set_attrs);
221 static ssize_t iio_show_scan_index(struct device *dev,
222 struct device_attribute *attr,
225 return sprintf(buf, "%u\n", to_iio_dev_attr(attr)->c->scan_index);
228 static ssize_t iio_show_fixed_type(struct device *dev,
229 struct device_attribute *attr,
232 struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
233 u8 type = this_attr->c->scan_type.endianness;
235 if (type == IIO_CPU) {
236 #ifdef __LITTLE_ENDIAN
242 if (this_attr->c->scan_type.repeat > 1)
243 return sprintf(buf, "%s:%c%d/%dX%d>>%u\n",
244 iio_endian_prefix[type],
245 this_attr->c->scan_type.sign,
246 this_attr->c->scan_type.realbits,
247 this_attr->c->scan_type.storagebits,
248 this_attr->c->scan_type.repeat,
249 this_attr->c->scan_type.shift);
251 return sprintf(buf, "%s:%c%d/%d>>%u\n",
252 iio_endian_prefix[type],
253 this_attr->c->scan_type.sign,
254 this_attr->c->scan_type.realbits,
255 this_attr->c->scan_type.storagebits,
256 this_attr->c->scan_type.shift);
259 static ssize_t iio_scan_el_show(struct device *dev,
260 struct device_attribute *attr,
264 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
266 /* Ensure ret is 0 or 1. */
267 ret = !!test_bit(to_iio_dev_attr(attr)->address,
268 indio_dev->buffer->scan_mask);
270 return sprintf(buf, "%d\n", ret);
273 /* Note NULL used as error indicator as it doesn't make sense. */
274 static const unsigned long *iio_scan_mask_match(const unsigned long *av_masks,
275 unsigned int masklength,
276 const unsigned long *mask,
279 if (bitmap_empty(mask, masklength))
283 if (bitmap_equal(mask, av_masks, masklength))
286 if (bitmap_subset(mask, av_masks, masklength))
289 av_masks += BITS_TO_LONGS(masklength);
294 static bool iio_validate_scan_mask(struct iio_dev *indio_dev,
295 const unsigned long *mask)
297 if (!indio_dev->setup_ops->validate_scan_mask)
300 return indio_dev->setup_ops->validate_scan_mask(indio_dev, mask);
304 * iio_scan_mask_set() - set particular bit in the scan mask
305 * @indio_dev: the iio device
306 * @buffer: the buffer whose scan mask we are interested in
307 * @bit: the bit to be set.
309 * Note that at this point we have no way of knowing what other
310 * buffers might request, hence this code only verifies that the
311 * individual buffers request is plausible.
313 static int iio_scan_mask_set(struct iio_dev *indio_dev,
314 struct iio_buffer *buffer, int bit)
316 const unsigned long *mask;
317 unsigned long *trialmask;
319 trialmask = kcalloc(BITS_TO_LONGS(indio_dev->masklength),
320 sizeof(*trialmask), GFP_KERNEL);
321 if (trialmask == NULL)
323 if (!indio_dev->masklength) {
324 WARN(1, "Trying to set scanmask prior to registering buffer\n");
325 goto err_invalid_mask;
327 bitmap_copy(trialmask, buffer->scan_mask, indio_dev->masklength);
328 set_bit(bit, trialmask);
330 if (!iio_validate_scan_mask(indio_dev, trialmask))
331 goto err_invalid_mask;
333 if (indio_dev->available_scan_masks) {
334 mask = iio_scan_mask_match(indio_dev->available_scan_masks,
335 indio_dev->masklength,
338 goto err_invalid_mask;
340 bitmap_copy(buffer->scan_mask, trialmask, indio_dev->masklength);
342 bitmap_free(trialmask);
347 bitmap_free(trialmask);
351 static int iio_scan_mask_clear(struct iio_buffer *buffer, int bit)
353 clear_bit(bit, buffer->scan_mask);
357 static int iio_scan_mask_query(struct iio_dev *indio_dev,
358 struct iio_buffer *buffer, int bit)
360 if (bit > indio_dev->masklength)
363 if (!buffer->scan_mask)
366 /* Ensure return value is 0 or 1. */
367 return !!test_bit(bit, buffer->scan_mask);
370 static ssize_t iio_scan_el_store(struct device *dev,
371 struct device_attribute *attr,
377 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
378 struct iio_buffer *buffer = indio_dev->buffer;
379 struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
381 ret = strtobool(buf, &state);
384 mutex_lock(&indio_dev->mlock);
385 if (iio_buffer_is_active(indio_dev->buffer)) {
389 ret = iio_scan_mask_query(indio_dev, buffer, this_attr->address);
393 ret = iio_scan_mask_clear(buffer, this_attr->address);
396 } else if (state && !ret) {
397 ret = iio_scan_mask_set(indio_dev, buffer, this_attr->address);
403 mutex_unlock(&indio_dev->mlock);
405 return ret < 0 ? ret : len;
409 static ssize_t iio_scan_el_ts_show(struct device *dev,
410 struct device_attribute *attr,
413 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
414 return sprintf(buf, "%d\n", indio_dev->buffer->scan_timestamp);
417 static ssize_t iio_scan_el_ts_store(struct device *dev,
418 struct device_attribute *attr,
423 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
426 ret = strtobool(buf, &state);
430 mutex_lock(&indio_dev->mlock);
431 if (iio_buffer_is_active(indio_dev->buffer)) {
435 indio_dev->buffer->scan_timestamp = state;
437 mutex_unlock(&indio_dev->mlock);
439 return ret ? ret : len;
442 static int iio_buffer_add_channel_sysfs(struct iio_dev *indio_dev,
443 const struct iio_chan_spec *chan)
445 int ret, attrcount = 0;
446 struct iio_buffer *buffer = indio_dev->buffer;
448 ret = __iio_add_chan_devattr("index",
450 &iio_show_scan_index,
455 &buffer->scan_el_dev_attr_list);
459 ret = __iio_add_chan_devattr("type",
461 &iio_show_fixed_type,
466 &buffer->scan_el_dev_attr_list);
470 if (chan->type != IIO_TIMESTAMP)
471 ret = __iio_add_chan_devattr("en",
478 &buffer->scan_el_dev_attr_list);
480 ret = __iio_add_chan_devattr("en",
482 &iio_scan_el_ts_show,
483 &iio_scan_el_ts_store,
487 &buffer->scan_el_dev_attr_list);
495 static ssize_t iio_buffer_read_length(struct device *dev,
496 struct device_attribute *attr,
499 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
500 struct iio_buffer *buffer = indio_dev->buffer;
502 return sprintf(buf, "%d\n", buffer->length);
505 static ssize_t iio_buffer_write_length(struct device *dev,
506 struct device_attribute *attr,
507 const char *buf, size_t len)
509 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
510 struct iio_buffer *buffer = indio_dev->buffer;
514 ret = kstrtouint(buf, 10, &val);
518 if (val == buffer->length)
521 mutex_lock(&indio_dev->mlock);
522 if (iio_buffer_is_active(indio_dev->buffer)) {
525 buffer->access->set_length(buffer, val);
530 if (buffer->length && buffer->length < buffer->watermark)
531 buffer->watermark = buffer->length;
533 mutex_unlock(&indio_dev->mlock);
535 return ret ? ret : len;
538 static ssize_t iio_buffer_show_enable(struct device *dev,
539 struct device_attribute *attr,
542 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
543 return sprintf(buf, "%d\n", iio_buffer_is_active(indio_dev->buffer));
546 static unsigned int iio_storage_bytes_for_si(struct iio_dev *indio_dev,
547 unsigned int scan_index)
549 const struct iio_chan_spec *ch;
552 ch = iio_find_channel_from_si(indio_dev, scan_index);
553 bytes = ch->scan_type.storagebits / 8;
554 if (ch->scan_type.repeat > 1)
555 bytes *= ch->scan_type.repeat;
559 static unsigned int iio_storage_bytes_for_timestamp(struct iio_dev *indio_dev)
561 return iio_storage_bytes_for_si(indio_dev,
562 indio_dev->scan_index_timestamp);
565 static int iio_compute_scan_bytes(struct iio_dev *indio_dev,
566 const unsigned long *mask, bool timestamp)
571 /* How much space will the demuxed element take? */
572 for_each_set_bit(i, mask,
573 indio_dev->masklength) {
574 length = iio_storage_bytes_for_si(indio_dev, i);
575 bytes = ALIGN(bytes, length);
580 length = iio_storage_bytes_for_timestamp(indio_dev);
581 bytes = ALIGN(bytes, length);
587 static void iio_buffer_activate(struct iio_dev *indio_dev,
588 struct iio_buffer *buffer)
590 iio_buffer_get(buffer);
591 list_add(&buffer->buffer_list, &indio_dev->buffer_list);
594 static void iio_buffer_deactivate(struct iio_buffer *buffer)
596 list_del_init(&buffer->buffer_list);
597 wake_up_interruptible(&buffer->pollq);
598 iio_buffer_put(buffer);
601 static void iio_buffer_deactivate_all(struct iio_dev *indio_dev)
603 struct iio_buffer *buffer, *_buffer;
605 list_for_each_entry_safe(buffer, _buffer,
606 &indio_dev->buffer_list, buffer_list)
607 iio_buffer_deactivate(buffer);
610 static int iio_buffer_enable(struct iio_buffer *buffer,
611 struct iio_dev *indio_dev)
613 if (!buffer->access->enable)
615 return buffer->access->enable(buffer, indio_dev);
618 static int iio_buffer_disable(struct iio_buffer *buffer,
619 struct iio_dev *indio_dev)
621 if (!buffer->access->disable)
623 return buffer->access->disable(buffer, indio_dev);
626 static void iio_buffer_update_bytes_per_datum(struct iio_dev *indio_dev,
627 struct iio_buffer *buffer)
631 if (!buffer->access->set_bytes_per_datum)
634 bytes = iio_compute_scan_bytes(indio_dev, buffer->scan_mask,
635 buffer->scan_timestamp);
637 buffer->access->set_bytes_per_datum(buffer, bytes);
640 static int iio_buffer_request_update(struct iio_dev *indio_dev,
641 struct iio_buffer *buffer)
645 iio_buffer_update_bytes_per_datum(indio_dev, buffer);
646 if (buffer->access->request_update) {
647 ret = buffer->access->request_update(buffer);
649 dev_dbg(&indio_dev->dev,
650 "Buffer not started: buffer parameter update failed (%d)\n",
659 static void iio_free_scan_mask(struct iio_dev *indio_dev,
660 const unsigned long *mask)
662 /* If the mask is dynamically allocated free it, otherwise do nothing */
663 if (!indio_dev->available_scan_masks)
667 struct iio_device_config {
669 unsigned int watermark;
670 const unsigned long *scan_mask;
671 unsigned int scan_bytes;
675 static int iio_verify_update(struct iio_dev *indio_dev,
676 struct iio_buffer *insert_buffer, struct iio_buffer *remove_buffer,
677 struct iio_device_config *config)
679 unsigned long *compound_mask;
680 const unsigned long *scan_mask;
681 bool strict_scanmask = false;
682 struct iio_buffer *buffer;
686 memset(config, 0, sizeof(*config));
687 config->watermark = ~0;
690 * If there is just one buffer and we are removing it there is nothing
693 if (remove_buffer && !insert_buffer &&
694 list_is_singular(&indio_dev->buffer_list))
697 modes = indio_dev->modes;
699 list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {
700 if (buffer == remove_buffer)
702 modes &= buffer->access->modes;
703 config->watermark = min(config->watermark, buffer->watermark);
707 modes &= insert_buffer->access->modes;
708 config->watermark = min(config->watermark,
709 insert_buffer->watermark);
712 /* Definitely possible for devices to support both of these. */
713 if ((modes & INDIO_BUFFER_TRIGGERED) && indio_dev->trig) {
714 config->mode = INDIO_BUFFER_TRIGGERED;
715 } else if (modes & INDIO_BUFFER_HARDWARE) {
717 * Keep things simple for now and only allow a single buffer to
718 * be connected in hardware mode.
720 if (insert_buffer && !list_empty(&indio_dev->buffer_list))
722 config->mode = INDIO_BUFFER_HARDWARE;
723 strict_scanmask = true;
724 } else if (modes & INDIO_BUFFER_SOFTWARE) {
725 config->mode = INDIO_BUFFER_SOFTWARE;
727 /* Can only occur on first buffer */
728 if (indio_dev->modes & INDIO_BUFFER_TRIGGERED)
729 dev_dbg(&indio_dev->dev, "Buffer not started: no trigger\n");
733 /* What scan mask do we actually have? */
734 compound_mask = bitmap_zalloc(indio_dev->masklength, GFP_KERNEL);
735 if (compound_mask == NULL)
738 scan_timestamp = false;
740 list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {
741 if (buffer == remove_buffer)
743 bitmap_or(compound_mask, compound_mask, buffer->scan_mask,
744 indio_dev->masklength);
745 scan_timestamp |= buffer->scan_timestamp;
749 bitmap_or(compound_mask, compound_mask,
750 insert_buffer->scan_mask, indio_dev->masklength);
751 scan_timestamp |= insert_buffer->scan_timestamp;
754 if (indio_dev->available_scan_masks) {
755 scan_mask = iio_scan_mask_match(indio_dev->available_scan_masks,
756 indio_dev->masklength,
759 bitmap_free(compound_mask);
760 if (scan_mask == NULL)
763 scan_mask = compound_mask;
766 config->scan_bytes = iio_compute_scan_bytes(indio_dev,
767 scan_mask, scan_timestamp);
768 config->scan_mask = scan_mask;
769 config->scan_timestamp = scan_timestamp;
775 * struct iio_demux_table - table describing demux memcpy ops
776 * @from: index to copy from
777 * @to: index to copy to
778 * @length: how many bytes to copy
779 * @l: list head used for management
781 struct iio_demux_table {
788 static void iio_buffer_demux_free(struct iio_buffer *buffer)
790 struct iio_demux_table *p, *q;
791 list_for_each_entry_safe(p, q, &buffer->demux_list, l) {
797 static int iio_buffer_add_demux(struct iio_buffer *buffer,
798 struct iio_demux_table **p, unsigned int in_loc, unsigned int out_loc,
802 if (*p && (*p)->from + (*p)->length == in_loc &&
803 (*p)->to + (*p)->length == out_loc) {
804 (*p)->length += length;
806 *p = kmalloc(sizeof(**p), GFP_KERNEL);
811 (*p)->length = length;
812 list_add_tail(&(*p)->l, &buffer->demux_list);
818 static int iio_buffer_update_demux(struct iio_dev *indio_dev,
819 struct iio_buffer *buffer)
821 int ret, in_ind = -1, out_ind, length;
822 unsigned in_loc = 0, out_loc = 0;
823 struct iio_demux_table *p = NULL;
825 /* Clear out any old demux */
826 iio_buffer_demux_free(buffer);
827 kfree(buffer->demux_bounce);
828 buffer->demux_bounce = NULL;
830 /* First work out which scan mode we will actually have */
831 if (bitmap_equal(indio_dev->active_scan_mask,
833 indio_dev->masklength))
836 /* Now we have the two masks, work from least sig and build up sizes */
837 for_each_set_bit(out_ind,
839 indio_dev->masklength) {
840 in_ind = find_next_bit(indio_dev->active_scan_mask,
841 indio_dev->masklength,
843 while (in_ind != out_ind) {
844 in_ind = find_next_bit(indio_dev->active_scan_mask,
845 indio_dev->masklength,
847 length = iio_storage_bytes_for_si(indio_dev, in_ind);
848 /* Make sure we are aligned */
849 in_loc = roundup(in_loc, length) + length;
851 length = iio_storage_bytes_for_si(indio_dev, in_ind);
852 out_loc = roundup(out_loc, length);
853 in_loc = roundup(in_loc, length);
854 ret = iio_buffer_add_demux(buffer, &p, in_loc, out_loc, length);
856 goto error_clear_mux_table;
860 /* Relies on scan_timestamp being last */
861 if (buffer->scan_timestamp) {
862 length = iio_storage_bytes_for_timestamp(indio_dev);
863 out_loc = roundup(out_loc, length);
864 in_loc = roundup(in_loc, length);
865 ret = iio_buffer_add_demux(buffer, &p, in_loc, out_loc, length);
867 goto error_clear_mux_table;
871 buffer->demux_bounce = kzalloc(out_loc, GFP_KERNEL);
872 if (buffer->demux_bounce == NULL) {
874 goto error_clear_mux_table;
878 error_clear_mux_table:
879 iio_buffer_demux_free(buffer);
884 static int iio_update_demux(struct iio_dev *indio_dev)
886 struct iio_buffer *buffer;
889 list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {
890 ret = iio_buffer_update_demux(indio_dev, buffer);
892 goto error_clear_mux_table;
896 error_clear_mux_table:
897 list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list)
898 iio_buffer_demux_free(buffer);
903 static int iio_enable_buffers(struct iio_dev *indio_dev,
904 struct iio_device_config *config)
906 struct iio_buffer *buffer;
909 indio_dev->active_scan_mask = config->scan_mask;
910 indio_dev->scan_timestamp = config->scan_timestamp;
911 indio_dev->scan_bytes = config->scan_bytes;
913 iio_update_demux(indio_dev);
916 if (indio_dev->setup_ops->preenable) {
917 ret = indio_dev->setup_ops->preenable(indio_dev);
919 dev_dbg(&indio_dev->dev,
920 "Buffer not started: buffer preenable failed (%d)\n", ret);
921 goto err_undo_config;
925 if (indio_dev->info->update_scan_mode) {
926 ret = indio_dev->info
927 ->update_scan_mode(indio_dev,
928 indio_dev->active_scan_mask);
930 dev_dbg(&indio_dev->dev,
931 "Buffer not started: update scan mode failed (%d)\n",
933 goto err_run_postdisable;
937 if (indio_dev->info->hwfifo_set_watermark)
938 indio_dev->info->hwfifo_set_watermark(indio_dev,
941 list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {
942 ret = iio_buffer_enable(buffer, indio_dev);
944 goto err_disable_buffers;
947 indio_dev->currentmode = config->mode;
949 if (indio_dev->setup_ops->postenable) {
950 ret = indio_dev->setup_ops->postenable(indio_dev);
952 dev_dbg(&indio_dev->dev,
953 "Buffer not started: postenable failed (%d)\n", ret);
954 goto err_disable_buffers;
961 list_for_each_entry_continue_reverse(buffer, &indio_dev->buffer_list,
963 iio_buffer_disable(buffer, indio_dev);
965 indio_dev->currentmode = INDIO_DIRECT_MODE;
966 if (indio_dev->setup_ops->postdisable)
967 indio_dev->setup_ops->postdisable(indio_dev);
969 indio_dev->active_scan_mask = NULL;
974 static int iio_disable_buffers(struct iio_dev *indio_dev)
976 struct iio_buffer *buffer;
980 /* Wind down existing buffers - iff there are any */
981 if (list_empty(&indio_dev->buffer_list))
985 * If things go wrong at some step in disable we still need to continue
986 * to perform the other steps, otherwise we leave the device in a
987 * inconsistent state. We return the error code for the first error we
991 if (indio_dev->setup_ops->predisable) {
992 ret2 = indio_dev->setup_ops->predisable(indio_dev);
997 list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {
998 ret2 = iio_buffer_disable(buffer, indio_dev);
1003 indio_dev->currentmode = INDIO_DIRECT_MODE;
1005 if (indio_dev->setup_ops->postdisable) {
1006 ret2 = indio_dev->setup_ops->postdisable(indio_dev);
1011 iio_free_scan_mask(indio_dev, indio_dev->active_scan_mask);
1012 indio_dev->active_scan_mask = NULL;
1017 static int __iio_update_buffers(struct iio_dev *indio_dev,
1018 struct iio_buffer *insert_buffer,
1019 struct iio_buffer *remove_buffer)
1021 struct iio_device_config new_config;
1024 ret = iio_verify_update(indio_dev, insert_buffer, remove_buffer,
1029 if (insert_buffer) {
1030 ret = iio_buffer_request_update(indio_dev, insert_buffer);
1032 goto err_free_config;
1035 ret = iio_disable_buffers(indio_dev);
1037 goto err_deactivate_all;
1040 iio_buffer_deactivate(remove_buffer);
1042 iio_buffer_activate(indio_dev, insert_buffer);
1044 /* If no buffers in list, we are done */
1045 if (list_empty(&indio_dev->buffer_list))
1048 ret = iio_enable_buffers(indio_dev, &new_config);
1050 goto err_deactivate_all;
1056 * We've already verified that the config is valid earlier. If things go
1057 * wrong in either enable or disable the most likely reason is an IO
1058 * error from the device. In this case there is no good recovery
1059 * strategy. Just make sure to disable everything and leave the device
1060 * in a sane state. With a bit of luck the device might come back to
1061 * life again later and userspace can try again.
1063 iio_buffer_deactivate_all(indio_dev);
1066 iio_free_scan_mask(indio_dev, new_config.scan_mask);
1070 int iio_update_buffers(struct iio_dev *indio_dev,
1071 struct iio_buffer *insert_buffer,
1072 struct iio_buffer *remove_buffer)
1076 if (insert_buffer == remove_buffer)
1079 mutex_lock(&indio_dev->info_exist_lock);
1080 mutex_lock(&indio_dev->mlock);
1082 if (insert_buffer && iio_buffer_is_active(insert_buffer))
1083 insert_buffer = NULL;
1085 if (remove_buffer && !iio_buffer_is_active(remove_buffer))
1086 remove_buffer = NULL;
1088 if (!insert_buffer && !remove_buffer) {
1093 if (indio_dev->info == NULL) {
1098 ret = __iio_update_buffers(indio_dev, insert_buffer, remove_buffer);
1101 mutex_unlock(&indio_dev->mlock);
1102 mutex_unlock(&indio_dev->info_exist_lock);
1106 EXPORT_SYMBOL_GPL(iio_update_buffers);
1108 void iio_disable_all_buffers(struct iio_dev *indio_dev)
1110 iio_disable_buffers(indio_dev);
1111 iio_buffer_deactivate_all(indio_dev);
1114 static ssize_t iio_buffer_store_enable(struct device *dev,
1115 struct device_attribute *attr,
1120 bool requested_state;
1121 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
1124 ret = strtobool(buf, &requested_state);
1128 mutex_lock(&indio_dev->mlock);
1130 /* Find out if it is in the list */
1131 inlist = iio_buffer_is_active(indio_dev->buffer);
1132 /* Already in desired state */
1133 if (inlist == requested_state)
1136 if (requested_state)
1137 ret = __iio_update_buffers(indio_dev,
1138 indio_dev->buffer, NULL);
1140 ret = __iio_update_buffers(indio_dev,
1141 NULL, indio_dev->buffer);
1144 mutex_unlock(&indio_dev->mlock);
1145 return (ret < 0) ? ret : len;
1148 static const char * const iio_scan_elements_group_name = "scan_elements";
1150 static ssize_t iio_buffer_show_watermark(struct device *dev,
1151 struct device_attribute *attr,
1154 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
1155 struct iio_buffer *buffer = indio_dev->buffer;
1157 return sprintf(buf, "%u\n", buffer->watermark);
1160 static ssize_t iio_buffer_store_watermark(struct device *dev,
1161 struct device_attribute *attr,
1165 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
1166 struct iio_buffer *buffer = indio_dev->buffer;
1170 ret = kstrtouint(buf, 10, &val);
1176 mutex_lock(&indio_dev->mlock);
1178 if (val > buffer->length) {
1183 if (iio_buffer_is_active(indio_dev->buffer)) {
1188 buffer->watermark = val;
1190 mutex_unlock(&indio_dev->mlock);
1192 return ret ? ret : len;
1195 static ssize_t iio_dma_show_data_available(struct device *dev,
1196 struct device_attribute *attr,
1199 struct iio_dev *indio_dev = dev_to_iio_dev(dev);
1202 bytes = iio_buffer_data_available(indio_dev->buffer);
1204 return sprintf(buf, "%zu\n", bytes);
1207 static DEVICE_ATTR(length, S_IRUGO | S_IWUSR, iio_buffer_read_length,
1208 iio_buffer_write_length);
1209 static struct device_attribute dev_attr_length_ro = __ATTR(length,
1210 S_IRUGO, iio_buffer_read_length, NULL);
1211 static DEVICE_ATTR(enable, S_IRUGO | S_IWUSR,
1212 iio_buffer_show_enable, iio_buffer_store_enable);
1213 static DEVICE_ATTR(watermark, S_IRUGO | S_IWUSR,
1214 iio_buffer_show_watermark, iio_buffer_store_watermark);
1215 static struct device_attribute dev_attr_watermark_ro = __ATTR(watermark,
1216 S_IRUGO, iio_buffer_show_watermark, NULL);
1217 static DEVICE_ATTR(data_available, S_IRUGO,
1218 iio_dma_show_data_available, NULL);
1220 static struct attribute *iio_buffer_attrs[] = {
1221 &dev_attr_length.attr,
1222 &dev_attr_enable.attr,
1223 &dev_attr_watermark.attr,
1224 &dev_attr_data_available.attr,
1227 int iio_buffer_alloc_sysfs_and_mask(struct iio_dev *indio_dev)
1229 struct iio_dev_attr *p;
1230 struct attribute **attr;
1231 struct iio_buffer *buffer = indio_dev->buffer;
1232 int ret, i, attrn, attrcount, attrcount_orig = 0;
1233 const struct iio_chan_spec *channels;
1235 channels = indio_dev->channels;
1237 int ml = indio_dev->masklength;
1239 for (i = 0; i < indio_dev->num_channels; i++)
1240 ml = max(ml, channels[i].scan_index + 1);
1241 indio_dev->masklength = ml;
1248 if (buffer->attrs) {
1249 while (buffer->attrs[attrcount] != NULL)
1253 attr = kcalloc(attrcount + ARRAY_SIZE(iio_buffer_attrs) + 1,
1254 sizeof(struct attribute *), GFP_KERNEL);
1258 memcpy(attr, iio_buffer_attrs, sizeof(iio_buffer_attrs));
1259 if (!buffer->access->set_length)
1260 attr[0] = &dev_attr_length_ro.attr;
1262 if (buffer->access->flags & INDIO_BUFFER_FLAG_FIXED_WATERMARK)
1263 attr[2] = &dev_attr_watermark_ro.attr;
1266 memcpy(&attr[ARRAY_SIZE(iio_buffer_attrs)], buffer->attrs,
1267 sizeof(struct attribute *) * attrcount);
1269 attr[attrcount + ARRAY_SIZE(iio_buffer_attrs)] = NULL;
1271 buffer->buffer_group.name = "buffer";
1272 buffer->buffer_group.attrs = attr;
1274 indio_dev->groups[indio_dev->groupcounter++] = &buffer->buffer_group;
1276 if (buffer->scan_el_attrs != NULL) {
1277 attr = buffer->scan_el_attrs->attrs;
1278 while (*attr++ != NULL)
1281 attrcount = attrcount_orig;
1282 INIT_LIST_HEAD(&buffer->scan_el_dev_attr_list);
1283 channels = indio_dev->channels;
1286 for (i = 0; i < indio_dev->num_channels; i++) {
1287 if (channels[i].scan_index < 0)
1290 ret = iio_buffer_add_channel_sysfs(indio_dev,
1293 goto error_cleanup_dynamic;
1295 if (channels[i].type == IIO_TIMESTAMP)
1296 indio_dev->scan_index_timestamp =
1297 channels[i].scan_index;
1299 if (indio_dev->masklength && buffer->scan_mask == NULL) {
1300 buffer->scan_mask = bitmap_zalloc(indio_dev->masklength,
1302 if (buffer->scan_mask == NULL) {
1304 goto error_cleanup_dynamic;
1309 buffer->scan_el_group.name = iio_scan_elements_group_name;
1311 buffer->scan_el_group.attrs = kcalloc(attrcount + 1,
1312 sizeof(buffer->scan_el_group.attrs[0]),
1314 if (buffer->scan_el_group.attrs == NULL) {
1316 goto error_free_scan_mask;
1318 if (buffer->scan_el_attrs)
1319 memcpy(buffer->scan_el_group.attrs, buffer->scan_el_attrs,
1320 sizeof(buffer->scan_el_group.attrs[0])*attrcount_orig);
1321 attrn = attrcount_orig;
1323 list_for_each_entry(p, &buffer->scan_el_dev_attr_list, l)
1324 buffer->scan_el_group.attrs[attrn++] = &p->dev_attr.attr;
1325 indio_dev->groups[indio_dev->groupcounter++] = &buffer->scan_el_group;
1329 error_free_scan_mask:
1330 bitmap_free(buffer->scan_mask);
1331 error_cleanup_dynamic:
1332 iio_free_chan_devattr_list(&buffer->scan_el_dev_attr_list);
1333 kfree(indio_dev->buffer->buffer_group.attrs);
1338 void iio_buffer_free_sysfs_and_mask(struct iio_dev *indio_dev)
1340 if (!indio_dev->buffer)
1343 bitmap_free(indio_dev->buffer->scan_mask);
1344 kfree(indio_dev->buffer->buffer_group.attrs);
1345 kfree(indio_dev->buffer->scan_el_group.attrs);
1346 iio_free_chan_devattr_list(&indio_dev->buffer->scan_el_dev_attr_list);
1350 * iio_validate_scan_mask_onehot() - Validates that exactly one channel is selected
1351 * @indio_dev: the iio device
1352 * @mask: scan mask to be checked
1354 * Return true if exactly one bit is set in the scan mask, false otherwise. It
1355 * can be used for devices where only one channel can be active for sampling at
1358 bool iio_validate_scan_mask_onehot(struct iio_dev *indio_dev,
1359 const unsigned long *mask)
1361 return bitmap_weight(mask, indio_dev->masklength) == 1;
1363 EXPORT_SYMBOL_GPL(iio_validate_scan_mask_onehot);
1365 static const void *iio_demux(struct iio_buffer *buffer,
1368 struct iio_demux_table *t;
1370 if (list_empty(&buffer->demux_list))
1372 list_for_each_entry(t, &buffer->demux_list, l)
1373 memcpy(buffer->demux_bounce + t->to,
1374 datain + t->from, t->length);
1376 return buffer->demux_bounce;
1379 static int iio_push_to_buffer(struct iio_buffer *buffer, const void *data)
1381 const void *dataout = iio_demux(buffer, data);
1384 ret = buffer->access->store_to(buffer, dataout);
1389 * We can't just test for watermark to decide if we wake the poll queue
1390 * because read may request less samples than the watermark.
1392 wake_up_interruptible_poll(&buffer->pollq, EPOLLIN | EPOLLRDNORM);
1397 * iio_push_to_buffers() - push to a registered buffer.
1398 * @indio_dev: iio_dev structure for device.
1401 int iio_push_to_buffers(struct iio_dev *indio_dev, const void *data)
1404 struct iio_buffer *buf;
1406 list_for_each_entry(buf, &indio_dev->buffer_list, buffer_list) {
1407 ret = iio_push_to_buffer(buf, data);
1414 EXPORT_SYMBOL_GPL(iio_push_to_buffers);
1417 * iio_buffer_release() - Free a buffer's resources
1418 * @ref: Pointer to the kref embedded in the iio_buffer struct
1420 * This function is called when the last reference to the buffer has been
1421 * dropped. It will typically free all resources allocated by the buffer. Do not
1422 * call this function manually, always use iio_buffer_put() when done using a
1425 static void iio_buffer_release(struct kref *ref)
1427 struct iio_buffer *buffer = container_of(ref, struct iio_buffer, ref);
1429 buffer->access->release(buffer);
1433 * iio_buffer_get() - Grab a reference to the buffer
1434 * @buffer: The buffer to grab a reference for, may be NULL
1436 * Returns the pointer to the buffer that was passed into the function.
1438 struct iio_buffer *iio_buffer_get(struct iio_buffer *buffer)
1441 kref_get(&buffer->ref);
1445 EXPORT_SYMBOL_GPL(iio_buffer_get);
1448 * iio_buffer_put() - Release the reference to the buffer
1449 * @buffer: The buffer to release the reference for, may be NULL
1451 void iio_buffer_put(struct iio_buffer *buffer)
1454 kref_put(&buffer->ref, iio_buffer_release);
1456 EXPORT_SYMBOL_GPL(iio_buffer_put);
1459 * iio_device_attach_buffer - Attach a buffer to a IIO device
1460 * @indio_dev: The device the buffer should be attached to
1461 * @buffer: The buffer to attach to the device
1463 * This function attaches a buffer to a IIO device. The buffer stays attached to
1464 * the device until the device is freed. The function should only be called at
1465 * most once per device.
1467 void iio_device_attach_buffer(struct iio_dev *indio_dev,
1468 struct iio_buffer *buffer)
1470 indio_dev->buffer = iio_buffer_get(buffer);
1472 EXPORT_SYMBOL_GPL(iio_device_attach_buffer);