kernel/dma/direct.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2018 Christoph Hellwig.
   4  *
   5  * DMA operations that map physical memory directly without using an IOMMU.
   6  */
   7 #include <linux/bootmem.h> /* for max_pfn */
   8 #include <linux/export.h>
   9 #include <linux/mm.h>
  10 #include <linux/dma-direct.h>
  11 #include <linux/scatterlist.h>
  12 #include <linux/dma-contiguous.h>
  13 #include <linux/dma-noncoherent.h>
  14 #include <linux/pfn.h>
  15 #include <linux/set_memory.h>
  16
  17 #define DIRECT_MAPPING_ERROR            0
  18
  19 /*
  20  * Most architectures use ZONE_DMA for the first 16 Megabytes, but
  21  * some use it for entirely different regions:
  22  */
  23 #ifndef ARCH_ZONE_DMA_BITS
  24 #define ARCH_ZONE_DMA_BITS 24
  25 #endif
  26
  27 /*
  28  * For AMD SEV all DMA must be to unencrypted addresses.
  29  */
  30 static inline bool force_dma_unencrypted(void)
  31 {
  32         return sev_active();
  33 }
  34
  35 static bool
  36 check_addr(struct device *dev, dma_addr_t dma_addr, size_t size,
  37                 const char *caller)
  38 {
  39         if (unlikely(dev && !dma_capable(dev, dma_addr, size))) {
  40                 if (!dev->dma_mask) {
  41                         dev_err(dev,
  42                                 "%s: call on device without dma_mask\n",
  43                                 caller);
  44                         return false;
  45                 }
  46
  47                 if (*dev->dma_mask >= DMA_BIT_MASK(32) || dev->bus_dma_mask) {
  48                         dev_err(dev,
  49                                 "%s: overflow %pad+%zu of device mask %llx bus mask %llx\n",
  50                                 caller, &dma_addr, size,
  51                                 *dev->dma_mask, dev->bus_dma_mask);
  52                 }
  53                 return false;
  54         }
  55         return true;
  56 }
  57
  58 static inline dma_addr_t phys_to_dma_direct(struct device *dev,
  59                 phys_addr_t phys)
  60 {
  61         if (force_dma_unencrypted())
  62                 return __phys_to_dma(dev, phys);
  63         return phys_to_dma(dev, phys);
  64 }
  65
  66 u64 dma_direct_get_required_mask(struct device *dev)
  67 {
  68         u64 max_dma = phys_to_dma_direct(dev, (max_pfn - 1) << PAGE_SHIFT);
  69
  70         if (dev->bus_dma_mask && dev->bus_dma_mask < max_dma)
  71                 max_dma = dev->bus_dma_mask;
  72
  73         return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
  74 }
  75
  76 static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
  77                 u64 *phys_mask)
  78 {
  79         if (dev->bus_dma_mask && dev->bus_dma_mask < dma_mask)
  80                 dma_mask = dev->bus_dma_mask;
  81
  82         if (force_dma_unencrypted())
  83                 *phys_mask = __dma_to_phys(dev, dma_mask);
  84         else
  85                 *phys_mask = dma_to_phys(dev, dma_mask);
  86
  87         /*
  88          * Optimistically try the zone that the physical address mask falls
  89          * into first.  If that returns memory that isn't actually addressable
  90          * we will fallback to the next lower zone and try again.
  91          *
  92          * Note that GFP_DMA32 and GFP_DMA are no ops without the corresponding
  93          * zones.
  94          */
  95         if (*phys_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS))
  96                 return GFP_DMA;
  97         if (*phys_mask <= DMA_BIT_MASK(32))
  98                 return GFP_DMA32;
  99         return 0;
 100 }
 101
 102 static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
 103 {
 104         return phys_to_dma_direct(dev, phys) + size - 1 <=
 105                         min_not_zero(dev->coherent_dma_mask, dev->bus_dma_mask);
 106 }
 107
 108 void *dma_direct_alloc_pages(struct device *dev, size_t size,
 109                 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
 110 {
 111         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 112         int page_order = get_order(size);
 113         struct page *page = NULL;
 114         u64 phys_mask;
 115         void *ret;
 116
 117         if (attrs & DMA_ATTR_NO_WARN)
 118                 gfp |= __GFP_NOWARN;
 119
 120         /* we always manually zero the memory once we are done: */
 121         gfp &= ~__GFP_ZERO;
 122         gfp |= __dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
 123                         &phys_mask);
 124 again:
 125         /* CMA can be used only in the context which permits sleeping */
 126         if (gfpflags_allow_blocking(gfp)) {
 127                 page = dma_alloc_from_contiguous(dev, count, page_order,
 128                                                  gfp & __GFP_NOWARN);
 129                 if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
 130                         dma_release_from_contiguous(dev, page, count);
 131                         page = NULL;
 132                 }
 133         }
 134         if (!page)
 135                 page = alloc_pages_node(dev_to_node(dev), gfp, page_order);
 136
 137         if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
 138                 __free_pages(page, page_order);
 139                 page = NULL;
 140
 141                 if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
 142                     phys_mask < DMA_BIT_MASK(64) &&
 143                     !(gfp & (GFP_DMA32 | GFP_DMA))) {
 144                         gfp |= GFP_DMA32;
 145                         goto again;
 146                 }
 147
 148                 if (IS_ENABLED(CONFIG_ZONE_DMA) &&
 149                     phys_mask < DMA_BIT_MASK(32) && !(gfp & GFP_DMA)) {
 150                         gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
 151                         goto again;
 152                 }
 153         }
 154
 155         if (!page)
 156                 return NULL;
 157         ret = page_address(page);
 158         if (force_dma_unencrypted()) {
 159                 set_memory_decrypted((unsigned long)ret, 1 << page_order);
 160                 *dma_handle = __phys_to_dma(dev, page_to_phys(page));
 161         } else {
 162                 *dma_handle = phys_to_dma(dev, page_to_phys(page));
 163         }
 164         memset(ret, 0, size);
 165         return ret;
 166 }
 167
 168 /*
 169  * NOTE: this function must never look at the dma_addr argument, because we want
 170  * to be able to use it as a helper for iommu implementations as well.
 171  */
 172 void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,
 173                 dma_addr_t dma_addr, unsigned long attrs)
 174 {
 175         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 176         unsigned int page_order = get_order(size);
 177
 178         if (force_dma_unencrypted())
 179                 set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);
 180         if (!dma_release_from_contiguous(dev, virt_to_page(cpu_addr), count))
 181                 free_pages((unsigned long)cpu_addr, page_order);
 182 }
 183
 184 void *dma_direct_alloc(struct device *dev, size_t size,
 185                 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
 186 {
 187         if (!dev_is_dma_coherent(dev))
 188                 return arch_dma_alloc(dev, size, dma_handle, gfp, attrs);
 189         return dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs);
 190 }
 191
 192 void dma_direct_free(struct device *dev, size_t size,
 193                 void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
 194 {
 195         if (!dev_is_dma_coherent(dev))
 196                 arch_dma_free(dev, size, cpu_addr, dma_addr, attrs);
 197         else
 198                 dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
 199 }
 200
 201 static void dma_direct_sync_single_for_device(struct device *dev,
 202                 dma_addr_t addr, size_t size, enum dma_data_direction dir)
 203 {
 204         if (dev_is_dma_coherent(dev))
 205                 return;
 206         arch_sync_dma_for_device(dev, dma_to_phys(dev, addr), size, dir);
 207 }
 208
 209 static void dma_direct_sync_sg_for_device(struct device *dev,
 210                 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
 211 {
 212         struct scatterlist *sg;
 213         int i;
 214
 215         if (dev_is_dma_coherent(dev))
 216                 return;
 217
 218         for_each_sg(sgl, sg, nents, i)
 219                 arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir);
 220 }
 221
 222 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
 223     defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
 224 static void dma_direct_sync_single_for_cpu(struct device *dev,
 225                 dma_addr_t addr, size_t size, enum dma_data_direction dir)
 226 {
 227         if (dev_is_dma_coherent(dev))
 228                 return;
 229         arch_sync_dma_for_cpu(dev, dma_to_phys(dev, addr), size, dir);
 230         arch_sync_dma_for_cpu_all(dev);
 231 }
 232
 233 static void dma_direct_sync_sg_for_cpu(struct device *dev,
 234                 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
 235 {
 236         struct scatterlist *sg;
 237         int i;
 238
 239         if (dev_is_dma_coherent(dev))
 240                 return;
 241
 242         for_each_sg(sgl, sg, nents, i)
 243                 arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir);
 244         arch_sync_dma_for_cpu_all(dev);
 245 }
 246
 247 static void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
 248                 size_t size, enum dma_data_direction dir, unsigned long attrs)
 249 {
 250         if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 251                 dma_direct_sync_single_for_cpu(dev, addr, size, dir);
 252 }
 253
 254 static void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
 255                 int nents, enum dma_data_direction dir, unsigned long attrs)
 256 {
 257         if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 258                 dma_direct_sync_sg_for_cpu(dev, sgl, nents, dir);
 259 }
 260 #endif
 261
 262 dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
 263                 unsigned long offset, size_t size, enum dma_data_direction dir,
 264                 unsigned long attrs)
 265 {
 266         phys_addr_t phys = page_to_phys(page) + offset;
 267         dma_addr_t dma_addr = phys_to_dma(dev, phys);
 268
 269         if (!check_addr(dev, dma_addr, size, __func__))
 270                 return DIRECT_MAPPING_ERROR;
 271
 272         if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 273                 dma_direct_sync_single_for_device(dev, dma_addr, size, dir);
 274         return dma_addr;
 275 }
 276
 277 int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
 278                 enum dma_data_direction dir, unsigned long attrs)
 279 {
 280         int i;
 281         struct scatterlist *sg;
 282
 283         for_each_sg(sgl, sg, nents, i) {
 284                 BUG_ON(!sg_page(sg));
 285
 286                 sg_dma_address(sg) = phys_to_dma(dev, sg_phys(sg));
 287                 if (!check_addr(dev, sg_dma_address(sg), sg->length, __func__))
 288                         return 0;
 289                 sg_dma_len(sg) = sg->length;
 290         }
 291
 292         if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 293                 dma_direct_sync_sg_for_device(dev, sgl, nents, dir);
 294         return nents;
 295 }
 296
 297 /*
 298  * Because 32-bit DMA masks are so common we expect every architecture to be
 299  * able to satisfy them - either by not supporting more physical memory, or by
 300  * providing a ZONE_DMA32.  If neither is the case, the architecture needs to
 301  * use an IOMMU instead of the direct mapping.
 302  */
 303 int dma_direct_supported(struct device *dev, u64 mask)
 304 {
 305         u64 min_mask;
 306
 307         if (IS_ENABLED(CONFIG_ZONE_DMA))
 308                 min_mask = DMA_BIT_MASK(ARCH_ZONE_DMA_BITS);
 309         else
 310                 min_mask = DMA_BIT_MASK(32);
 311
 312         min_mask = min_t(u64, min_mask, (max_pfn - 1) << PAGE_SHIFT);
 313
 314         return mask >= phys_to_dma(dev, min_mask);
 315 }
 316
 317 int dma_direct_mapping_error(struct device *dev, dma_addr_t dma_addr)
 318 {
 319         return dma_addr == DIRECT_MAPPING_ERROR;
 320 }
 321
 322 const struct dma_map_ops dma_direct_ops = {
 323         .alloc                  = dma_direct_alloc,
 324         .free                   = dma_direct_free,
 325         .map_page               = dma_direct_map_page,
 326         .map_sg                 = dma_direct_map_sg,
 327 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE)
 328         .sync_single_for_device = dma_direct_sync_single_for_device,
 329         .sync_sg_for_device     = dma_direct_sync_sg_for_device,
 330 #endif
 331 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
 332     defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
 333         .sync_single_for_cpu    = dma_direct_sync_single_for_cpu,
 334         .sync_sg_for_cpu        = dma_direct_sync_sg_for_cpu,
 335         .unmap_page             = dma_direct_unmap_page,
 336         .unmap_sg               = dma_direct_unmap_sg,
 337 #endif
 338         .get_required_mask      = dma_direct_get_required_mask,
 339         .dma_supported          = dma_direct_supported,
 340         .mapping_error          = dma_direct_mapping_error,
 341         .cache_sync             = arch_dma_cache_sync,
 342 };
 343 EXPORT_SYMBOL(dma_direct_ops);