2 * Based on linux/arch/arm/mm/dma-mapping.c
4 * Copyright (C) 2000-2004 Russell King
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
12 #include <linux/export.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/scatterlist.h>
17 #include <asm/cachetype.h>
18 #include <asm/cacheflush.h>
19 #include <asm/outercache.h>
25 * dma_noop_ops is used if
27 * - cpu is v7m w/o cache support
28 * - device is coherent
29 * otherwise arm_nommu_dma_ops is used.
31 * arm_nommu_dma_ops rely on consistent DMA memory (please, refer to
32 * [1] on how to declare such memory).
34 * [1] Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt
37 static void *arm_nommu_dma_alloc(struct device *dev, size_t size,
38 dma_addr_t *dma_handle, gfp_t gfp,
42 const struct dma_map_ops *ops = &dma_noop_ops;
45 * We are here because:
46 * - no consistent DMA region has been defined, so we can't
48 * - there is no space left in consistent DMA region, so we
49 * only can fallback to generic allocator if we are
50 * advertised that consistency is not required.
53 if (attrs & DMA_ATTR_NON_CONSISTENT)
54 return ops->alloc(dev, size, dma_handle, gfp, attrs);
60 static void arm_nommu_dma_free(struct device *dev, size_t size,
61 void *cpu_addr, dma_addr_t dma_addr,
64 const struct dma_map_ops *ops = &dma_noop_ops;
66 if (attrs & DMA_ATTR_NON_CONSISTENT)
67 ops->free(dev, size, cpu_addr, dma_addr, attrs);
74 static void __dma_page_cpu_to_dev(phys_addr_t paddr, size_t size,
75 enum dma_data_direction dir)
77 dmac_map_area(__va(paddr), size, dir);
79 if (dir == DMA_FROM_DEVICE)
80 outer_inv_range(paddr, paddr + size);
82 outer_clean_range(paddr, paddr + size);
85 static void __dma_page_dev_to_cpu(phys_addr_t paddr, size_t size,
86 enum dma_data_direction dir)
88 if (dir != DMA_TO_DEVICE) {
89 outer_inv_range(paddr, paddr + size);
90 dmac_unmap_area(__va(paddr), size, dir);
94 static dma_addr_t arm_nommu_dma_map_page(struct device *dev, struct page *page,
95 unsigned long offset, size_t size,
96 enum dma_data_direction dir,
99 dma_addr_t handle = page_to_phys(page) + offset;
101 __dma_page_cpu_to_dev(handle, size, dir);
106 static void arm_nommu_dma_unmap_page(struct device *dev, dma_addr_t handle,
107 size_t size, enum dma_data_direction dir,
110 __dma_page_dev_to_cpu(handle, size, dir);
114 static int arm_nommu_dma_map_sg(struct device *dev, struct scatterlist *sgl,
115 int nents, enum dma_data_direction dir,
119 struct scatterlist *sg;
121 for_each_sg(sgl, sg, nents, i) {
122 sg_dma_address(sg) = sg_phys(sg);
123 sg_dma_len(sg) = sg->length;
124 __dma_page_cpu_to_dev(sg_dma_address(sg), sg_dma_len(sg), dir);
130 static void arm_nommu_dma_unmap_sg(struct device *dev, struct scatterlist *sgl,
131 int nents, enum dma_data_direction dir,
134 struct scatterlist *sg;
137 for_each_sg(sgl, sg, nents, i)
138 __dma_page_dev_to_cpu(sg_dma_address(sg), sg_dma_len(sg), dir);
141 static void arm_nommu_dma_sync_single_for_device(struct device *dev,
142 dma_addr_t handle, size_t size, enum dma_data_direction dir)
144 __dma_page_cpu_to_dev(handle, size, dir);
147 static void arm_nommu_dma_sync_single_for_cpu(struct device *dev,
148 dma_addr_t handle, size_t size, enum dma_data_direction dir)
150 __dma_page_cpu_to_dev(handle, size, dir);
153 static void arm_nommu_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
154 int nents, enum dma_data_direction dir)
156 struct scatterlist *sg;
159 for_each_sg(sgl, sg, nents, i)
160 __dma_page_cpu_to_dev(sg_dma_address(sg), sg_dma_len(sg), dir);
163 static void arm_nommu_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl,
164 int nents, enum dma_data_direction dir)
166 struct scatterlist *sg;
169 for_each_sg(sgl, sg, nents, i)
170 __dma_page_dev_to_cpu(sg_dma_address(sg), sg_dma_len(sg), dir);
173 const struct dma_map_ops arm_nommu_dma_ops = {
174 .alloc = arm_nommu_dma_alloc,
175 .free = arm_nommu_dma_free,
176 .map_page = arm_nommu_dma_map_page,
177 .unmap_page = arm_nommu_dma_unmap_page,
178 .map_sg = arm_nommu_dma_map_sg,
179 .unmap_sg = arm_nommu_dma_unmap_sg,
180 .sync_single_for_device = arm_nommu_dma_sync_single_for_device,
181 .sync_single_for_cpu = arm_nommu_dma_sync_single_for_cpu,
182 .sync_sg_for_device = arm_nommu_dma_sync_sg_for_device,
183 .sync_sg_for_cpu = arm_nommu_dma_sync_sg_for_cpu,
185 EXPORT_SYMBOL(arm_nommu_dma_ops);
187 static const struct dma_map_ops *arm_nommu_get_dma_map_ops(bool coherent)
189 return coherent ? &dma_noop_ops : &arm_nommu_dma_ops;
192 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
193 const struct iommu_ops *iommu, bool coherent)
195 const struct dma_map_ops *dma_ops;
197 if (IS_ENABLED(CONFIG_CPU_V7M)) {
199 * Cache support for v7m is optional, so can be treated as
200 * coherent if no cache has been detected. Note that it is not
201 * enough to check if MPU is in use or not since in absense of
202 * MPU system memory map is used.
204 dev->archdata.dma_coherent = (cacheid) ? coherent : true;
207 * Assume coherent DMA in case MMU/MPU has not been set up.
209 dev->archdata.dma_coherent = (get_cr() & CR_M) ? coherent : true;
212 dma_ops = arm_nommu_get_dma_map_ops(dev->archdata.dma_coherent);
214 set_dma_ops(dev, dma_ops);
217 void arch_teardown_dma_ops(struct device *dev)
221 #define PREALLOC_DMA_DEBUG_ENTRIES 4096
223 static int __init dma_debug_do_init(void)
225 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
228 core_initcall(dma_debug_do_init);