132-mips_inline_dma_ops.patch 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778
  1. From 2c58080407554e1bac8fd50d23cb02420524caed Mon Sep 17 00:00:00 2001
  2. From: Felix Fietkau <nbd@nbd.name>
  3. Date: Mon, 12 Aug 2013 12:50:22 +0200
  4. Subject: [PATCH] MIPS: partially inline dma ops
  5. Several DMA ops are no-op on many platforms, and the indirection through
  6. the mips_dma_map_ops function table is causing the compiler to emit
  7. unnecessary code.
  8. Inlining visibly improves network performance in my tests (on a 24Kc
  9. based system), and also slightly reduces code size of a few drivers.
  10. Signed-off-by: Felix Fietkau <nbd@nbd.name>
  11. ---
  12. arch/mips/Kconfig | 4 +
  13. arch/mips/include/asm/dma-mapping.h | 360 +++++++++++++++++++++++++++++++++++-
  14. arch/mips/mm/dma-default.c | 163 ++--------------
  15. 3 files changed, 373 insertions(+), 154 deletions(-)
  16. --- a/arch/mips/Kconfig
  17. +++ b/arch/mips/Kconfig
  18. @@ -1620,6 +1620,7 @@ config CPU_CAVIUM_OCTEON
  19. select USB_EHCI_BIG_ENDIAN_MMIO if CPU_BIG_ENDIAN
  20. select USB_OHCI_BIG_ENDIAN_MMIO if CPU_BIG_ENDIAN
  21. select MIPS_L1_CACHE_SHIFT_7
  22. + select SYS_HAS_DMA_OPS
  23. help
  24. The Cavium Octeon processor is a highly integrated chip containing
  25. many ethernet hardware widgets for networking tasks. The processor
  26. @@ -1915,6 +1916,9 @@ config MIPS_MALTA_PM
  27. bool
  28. default y
  29. +config SYS_HAS_DMA_OPS
  30. + bool
  31. +
  32. #
  33. # CPU may reorder R->R, R->W, W->R, W->W
  34. # Reordering beyond LL and SC is handled in WEAK_REORDERING_BEYOND_LLSC
  35. --- a/arch/mips/include/asm/dma-mapping.h
  36. +++ b/arch/mips/include/asm/dma-mapping.h
  37. @@ -1,9 +1,16 @@
  38. #ifndef _ASM_DMA_MAPPING_H
  39. #define _ASM_DMA_MAPPING_H
  40. +#include <linux/kmemcheck.h>
  41. +#include <linux/bug.h>
  42. #include <linux/scatterlist.h>
  43. +#include <linux/dma-debug.h>
  44. +#include <linux/dma-attrs.h>
  45. +
  46. #include <asm/dma-coherence.h>
  47. #include <asm/cache.h>
  48. +#include <asm/cpu-type.h>
  49. +#include <asm-generic/dma-coherent.h>
  50. #ifndef CONFIG_SGI_IP27 /* Kludge to fix 2.6.39 build for IP27 */
  51. #include <dma-coherence.h>
  52. @@ -11,12 +18,53 @@
  53. extern struct dma_map_ops *mips_dma_map_ops;
  54. +void __dma_sync(struct page *page, unsigned long offset, size_t size,
  55. + enum dma_data_direction direction);
  56. +void *mips_dma_alloc_coherent(struct device *dev, size_t size,
  57. + dma_addr_t *dma_handle, gfp_t gfp,
  58. + struct dma_attrs *attrs);
  59. +void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
  60. + dma_addr_t dma_handle, struct dma_attrs *attrs);
  61. +
  62. static inline struct dma_map_ops *get_dma_ops(struct device *dev)
  63. {
  64. +#ifdef CONFIG_SYS_HAS_DMA_OPS
  65. if (dev && dev->archdata.dma_ops)
  66. return dev->archdata.dma_ops;
  67. else
  68. return mips_dma_map_ops;
  69. +#else
  70. + return NULL;
  71. +#endif
  72. +}
  73. +
  74. +/*
  75. + * The affected CPUs below in 'cpu_needs_post_dma_flush()' can
  76. + * speculatively fill random cachelines with stale data at any time,
  77. + * requiring an extra flush post-DMA.
  78. + *
  79. + * Warning on the terminology - Linux calls an uncached area coherent;
  80. + * MIPS terminology calls memory areas with hardware maintained coherency
  81. + * coherent.
  82. + *
  83. + * Note that the R14000 and R16000 should also be checked for in this
  84. + * condition. However this function is only called on non-I/O-coherent
  85. + * systems and only the R10000 and R12000 are used in such systems, the
  86. + * SGI IP28 Indigo² rsp. SGI IP32 aka O2.
  87. + */
  88. +static inline int cpu_needs_post_dma_flush(struct device *dev)
  89. +{
  90. + return !plat_device_is_coherent(dev) &&
  91. + (boot_cpu_type() == CPU_R10000 ||
  92. + boot_cpu_type() == CPU_R12000 ||
  93. + boot_cpu_type() == CPU_BMIPS5000);
  94. +}
  95. +
  96. +static inline struct page *dma_addr_to_page(struct device *dev,
  97. + dma_addr_t dma_addr)
  98. +{
  99. + return pfn_to_page(
  100. + plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
  101. }
  102. static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
  103. @@ -29,9 +77,399 @@ static inline bool dma_capable(struct de
  104. static inline void dma_mark_clean(void *addr, size_t size) {}
  105. -#include <asm-generic/dma-mapping-common.h>
  106. +static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
  107. + size_t size,
  108. + enum dma_data_direction dir,
  109. + struct dma_attrs *attrs)
  110. +{
  111. + struct dma_map_ops *ops = get_dma_ops(dev);
  112. + unsigned long offset = (unsigned long)ptr & ~PAGE_MASK;
  113. + struct page *page = virt_to_page(ptr);
  114. + dma_addr_t addr;
  115. +
  116. + kmemcheck_mark_initialized(ptr, size);
  117. + BUG_ON(!valid_dma_direction(dir));
  118. + if (ops) {
  119. + addr = ops->map_page(dev, page, offset, size, dir, attrs);
  120. + } else {
  121. + if (!plat_device_is_coherent(dev))
  122. + __dma_sync(page, offset, size, dir);
  123. +
  124. + addr = plat_map_dma_mem_page(dev, page) + offset;
  125. + }
  126. + debug_dma_map_page(dev, page, offset, size, dir, addr, true);
  127. + return addr;
  128. +}
  129. +
  130. +static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr,
  131. + size_t size,
  132. + enum dma_data_direction dir,
  133. + struct dma_attrs *attrs)
  134. +{
  135. + struct dma_map_ops *ops = get_dma_ops(dev);
  136. +
  137. + BUG_ON(!valid_dma_direction(dir));
  138. + if (ops) {
  139. + ops->unmap_page(dev, addr, size, dir, attrs);
  140. + } else {
  141. + if (cpu_needs_post_dma_flush(dev))
  142. + __dma_sync(dma_addr_to_page(dev, addr),
  143. + addr & ~PAGE_MASK, size, dir);
  144. + plat_post_dma_flush(dev);
  145. + plat_unmap_dma_mem(dev, addr, size, dir);
  146. + }
  147. + debug_dma_unmap_page(dev, addr, size, dir, true);
  148. +}
  149. +
  150. +/*
  151. + * dma_maps_sg_attrs returns 0 on error and > 0 on success.
  152. + * It should never return a value < 0.
  153. + */
  154. +static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
  155. + int nents, enum dma_data_direction dir,
  156. + struct dma_attrs *attrs)
  157. +{
  158. + struct dma_map_ops *ops = get_dma_ops(dev);
  159. + int i, ents;
  160. + struct scatterlist *s;
  161. +
  162. + for_each_sg(sg, s, nents, i)
  163. + kmemcheck_mark_initialized(sg_virt(s), s->length);
  164. + BUG_ON(!valid_dma_direction(dir));
  165. + if (ops) {
  166. + ents = ops->map_sg(dev, sg, nents, dir, attrs);
  167. + } else {
  168. + for_each_sg(sg, s, nents, i) {
  169. + struct page *page = sg_page(s);
  170. +
  171. + if (!plat_device_is_coherent(dev))
  172. + __dma_sync(page, s->offset, s->length, dir);
  173. +#ifdef CONFIG_NEED_SG_DMA_LENGTH
  174. + s->dma_length = s->length;
  175. +#endif
  176. + s->dma_address =
  177. + plat_map_dma_mem_page(dev, page) + s->offset;
  178. + }
  179. + ents = nents;
  180. + }
  181. + BUG_ON(ents < 0);
  182. + debug_dma_map_sg(dev, sg, nents, ents, dir);
  183. +
  184. + return ents;
  185. +}
  186. +
  187. +static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
  188. + int nents, enum dma_data_direction dir,
  189. + struct dma_attrs *attrs)
  190. +{
  191. + struct dma_map_ops *ops = get_dma_ops(dev);
  192. + struct scatterlist *s;
  193. + int i;
  194. +
  195. + BUG_ON(!valid_dma_direction(dir));
  196. + debug_dma_unmap_sg(dev, sg, nents, dir);
  197. + if (ops) {
  198. + ops->unmap_sg(dev, sg, nents, dir, attrs);
  199. + return;
  200. + }
  201. + for_each_sg(sg, s, nents, i) {
  202. + if (!plat_device_is_coherent(dev) && dir != DMA_TO_DEVICE)
  203. + __dma_sync(sg_page(s), s->offset, s->length, dir);
  204. + plat_unmap_dma_mem(dev, s->dma_address, s->length, dir);
  205. + }
  206. +}
  207. +
  208. +static inline dma_addr_t dma_map_page(struct device *dev, struct page *page,
  209. + size_t offset, size_t size,
  210. + enum dma_data_direction dir)
  211. +{
  212. + struct dma_map_ops *ops = get_dma_ops(dev);
  213. + dma_addr_t addr;
  214. +
  215. + kmemcheck_mark_initialized(page_address(page) + offset, size);
  216. + BUG_ON(!valid_dma_direction(dir));
  217. + if (ops) {
  218. + addr = ops->map_page(dev, page, offset, size, dir, NULL);
  219. + } else {
  220. + if (!plat_device_is_coherent(dev))
  221. + __dma_sync(page, offset, size, dir);
  222. +
  223. + addr = plat_map_dma_mem_page(dev, page) + offset;
  224. + }
  225. + debug_dma_map_page(dev, page, offset, size, dir, addr, false);
  226. +
  227. + return addr;
  228. +}
  229. +
  230. +static inline void dma_unmap_page(struct device *dev, dma_addr_t addr,
  231. + size_t size, enum dma_data_direction dir)
  232. +{
  233. + struct dma_map_ops *ops = get_dma_ops(dev);
  234. +
  235. + BUG_ON(!valid_dma_direction(dir));
  236. + if (ops) {
  237. + ops->unmap_page(dev, addr, size, dir, NULL);
  238. + } else {
  239. + if (cpu_needs_post_dma_flush(dev))
  240. + __dma_sync(dma_addr_to_page(dev, addr),
  241. + addr & ~PAGE_MASK, size, dir);
  242. + plat_post_dma_flush(dev);
  243. + plat_unmap_dma_mem(dev, addr, size, dir);
  244. + }
  245. + debug_dma_unmap_page(dev, addr, size, dir, false);
  246. +}
  247. +
  248. +static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
  249. + size_t size,
  250. + enum dma_data_direction dir)
  251. +{
  252. + struct dma_map_ops *ops = get_dma_ops(dev);
  253. +
  254. + BUG_ON(!valid_dma_direction(dir));
  255. + if (ops) {
  256. + ops->sync_single_for_cpu(dev, addr, size, dir);
  257. + } else {
  258. + if (cpu_needs_post_dma_flush(dev))
  259. + __dma_sync(dma_addr_to_page(dev, addr),
  260. + addr & ~PAGE_MASK, size, dir);
  261. + plat_post_dma_flush(dev);
  262. + }
  263. + debug_dma_sync_single_for_cpu(dev, addr, size, dir);
  264. +}
  265. +
  266. +static inline void dma_sync_single_for_device(struct device *dev,
  267. + dma_addr_t addr, size_t size,
  268. + enum dma_data_direction dir)
  269. +{
  270. + struct dma_map_ops *ops = get_dma_ops(dev);
  271. +
  272. + BUG_ON(!valid_dma_direction(dir));
  273. + if (ops)
  274. + ops->sync_single_for_device(dev, addr, size, dir);
  275. + else if (!plat_device_is_coherent(dev))
  276. + __dma_sync(dma_addr_to_page(dev, addr),
  277. + addr & ~PAGE_MASK, size, dir);
  278. + debug_dma_sync_single_for_device(dev, addr, size, dir);
  279. +}
  280. +
  281. +static inline void dma_sync_single_range_for_cpu(struct device *dev,
  282. + dma_addr_t addr,
  283. + unsigned long offset,
  284. + size_t size,
  285. + enum dma_data_direction dir)
  286. +{
  287. + const struct dma_map_ops *ops = get_dma_ops(dev);
  288. +
  289. + BUG_ON(!valid_dma_direction(dir));
  290. + if (ops) {
  291. + ops->sync_single_for_cpu(dev, addr + offset, size, dir);
  292. + } else {
  293. + if (cpu_needs_post_dma_flush(dev))
  294. + __dma_sync(dma_addr_to_page(dev, addr + offset),
  295. + (addr + offset) & ~PAGE_MASK, size, dir);
  296. + plat_post_dma_flush(dev);
  297. + }
  298. +
  299. + debug_dma_sync_single_range_for_cpu(dev, addr, offset, size, dir);
  300. +}
  301. +
  302. +static inline void dma_sync_single_range_for_device(struct device *dev,
  303. + dma_addr_t addr,
  304. + unsigned long offset,
  305. + size_t size,
  306. + enum dma_data_direction dir)
  307. +{
  308. + const struct dma_map_ops *ops = get_dma_ops(dev);
  309. +
  310. + BUG_ON(!valid_dma_direction(dir));
  311. + if (ops)
  312. + ops->sync_single_for_device(dev, addr + offset, size, dir);
  313. + else if (!plat_device_is_coherent(dev))
  314. + __dma_sync(dma_addr_to_page(dev, addr + offset),
  315. + (addr + offset) & ~PAGE_MASK, size, dir);
  316. + debug_dma_sync_single_range_for_device(dev, addr, offset, size, dir);
  317. +}
  318. +
  319. +static inline void
  320. +dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
  321. + int nelems, enum dma_data_direction dir)
  322. +{
  323. + struct dma_map_ops *ops = get_dma_ops(dev);
  324. + struct scatterlist *s;
  325. + int i;
  326. +
  327. + BUG_ON(!valid_dma_direction(dir));
  328. + if (ops) {
  329. + ops->sync_sg_for_cpu(dev, sg, nelems, dir);
  330. + } else if (cpu_needs_post_dma_flush(dev)) {
  331. + for_each_sg(sg, s, nelems, i)
  332. + __dma_sync(sg_page(s), s->offset, s->length, dir);
  333. + }
  334. + plat_post_dma_flush(dev);
  335. + debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
  336. +}
  337. +
  338. +static inline void
  339. +dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
  340. + int nelems, enum dma_data_direction dir)
  341. +{
  342. + struct dma_map_ops *ops = get_dma_ops(dev);
  343. + struct scatterlist *s;
  344. + int i;
  345. +
  346. + BUG_ON(!valid_dma_direction(dir));
  347. + if (ops) {
  348. + ops->sync_sg_for_device(dev, sg, nelems, dir);
  349. + } else if (!plat_device_is_coherent(dev)) {
  350. + for_each_sg(sg, s, nelems, i)
  351. + __dma_sync(sg_page(s), s->offset, s->length, dir);
  352. + }
  353. + debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
  354. +
  355. +}
  356. +
  357. +#define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, NULL)
  358. +#define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, NULL)
  359. +#define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, NULL)
  360. +#define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, NULL)
  361. +
  362. +extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
  363. + void *cpu_addr, dma_addr_t dma_addr, size_t size);
  364. +
  365. +/**
  366. + * dma_mmap_attrs - map a coherent DMA allocation into user space
  367. + * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
  368. + * @vma: vm_area_struct describing requested user mapping
  369. + * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs
  370. + * @handle: device-view address returned from dma_alloc_attrs
  371. + * @size: size of memory originally requested in dma_alloc_attrs
  372. + * @attrs: attributes of mapping properties requested in dma_alloc_attrs
  373. + *
  374. + * Map a coherent DMA buffer previously allocated by dma_alloc_attrs
  375. + * into user space. The coherent DMA buffer must not be freed by the
  376. + * driver until the user space mapping has been released.
  377. + */
  378. +static inline int
  379. +dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr,
  380. + dma_addr_t dma_addr, size_t size, struct dma_attrs *attrs)
  381. +{
  382. + struct dma_map_ops *ops = get_dma_ops(dev);
  383. + BUG_ON(!ops);
  384. + if (ops && ops->mmap)
  385. + return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
  386. + return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
  387. +}
  388. +
  389. +#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, NULL)
  390. +
  391. +int
  392. +dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
  393. + void *cpu_addr, dma_addr_t dma_addr, size_t size);
  394. +
  395. +static inline int
  396. +dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr,
  397. + dma_addr_t dma_addr, size_t size, struct dma_attrs *attrs)
  398. +{
  399. + struct dma_map_ops *ops = get_dma_ops(dev);
  400. + BUG_ON(!ops);
  401. + if (ops && ops->get_sgtable)
  402. + return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
  403. + attrs);
  404. + return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size);
  405. +}
  406. +
  407. +#define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, NULL)
  408. +
  409. +static inline int dma_supported(struct device *dev, u64 mask)
  410. +{
  411. + struct dma_map_ops *ops = get_dma_ops(dev);
  412. + if (ops)
  413. + return ops->dma_supported(dev, mask);
  414. + return plat_dma_supported(dev, mask);
  415. +}
  416. +
  417. +static inline int dma_mapping_error(struct device *dev, u64 mask)
  418. +{
  419. + struct dma_map_ops *ops = get_dma_ops(dev);
  420. +
  421. + debug_dma_mapping_error(dev, mask);
  422. + if (ops)
  423. + return ops->mapping_error(dev, mask);
  424. + return 0;
  425. +}
  426. +
  427. +static inline int
  428. +dma_set_mask(struct device *dev, u64 mask)
  429. +{
  430. + struct dma_map_ops *ops = get_dma_ops(dev);
  431. +
  432. + if(!dev->dma_mask || !dma_supported(dev, mask))
  433. + return -EIO;
  434. +
  435. + if (ops && ops->set_dma_mask)
  436. + return ops->set_dma_mask(dev, mask);
  437. +
  438. + *dev->dma_mask = mask;
  439. +
  440. + return 0;
  441. +}
  442. extern void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
  443. enum dma_data_direction direction);
  444. +#define dma_alloc_coherent(d,s,h,f) dma_alloc_attrs(d,s,h,f,NULL)
  445. +
  446. +static inline void *dma_alloc_attrs(struct device *dev, size_t size,
  447. + dma_addr_t *dma_handle, gfp_t gfp,
  448. + struct dma_attrs *attrs)
  449. +{
  450. + void *ret;
  451. + struct dma_map_ops *ops = get_dma_ops(dev);
  452. +
  453. + if (ops)
  454. + ret = ops->alloc(dev, size, dma_handle, gfp, attrs);
  455. + else
  456. + ret = mips_dma_alloc_coherent(dev, size, dma_handle, gfp,
  457. + attrs);
  458. +
  459. + debug_dma_alloc_coherent(dev, size, *dma_handle, ret);
  460. +
  461. + return ret;
  462. +}
  463. +
  464. +#define dma_free_coherent(d,s,c,h) dma_free_attrs(d,s,c,h,NULL)
  465. +
  466. +static inline void dma_free_attrs(struct device *dev, size_t size,
  467. + void *vaddr, dma_addr_t dma_handle,
  468. + struct dma_attrs *attrs)
  469. +{
  470. + struct dma_map_ops *ops = get_dma_ops(dev);
  471. +
  472. + if (ops)
  473. + ops->free(dev, size, vaddr, dma_handle, attrs);
  474. + else
  475. + mips_dma_free_coherent(dev, size, vaddr, dma_handle, attrs);
  476. +
  477. + debug_dma_free_coherent(dev, size, vaddr, dma_handle);
  478. +}
  479. +
  480. +static inline void *dma_alloc_noncoherent(struct device *dev, size_t size,
  481. + dma_addr_t *dma_handle, gfp_t gfp)
  482. +{
  483. + DEFINE_DMA_ATTRS(attrs);
  484. +
  485. + dma_set_attr(DMA_ATTR_NON_CONSISTENT, &attrs);
  486. + return dma_alloc_attrs(dev, size, dma_handle, gfp, &attrs);
  487. +}
  488. +
  489. +static inline void dma_free_noncoherent(struct device *dev, size_t size,
  490. + void *cpu_addr, dma_addr_t dma_handle)
  491. +{
  492. + DEFINE_DMA_ATTRS(attrs);
  493. +
  494. + dma_set_attr(DMA_ATTR_NON_CONSISTENT, &attrs);
  495. + dma_free_attrs(dev, size, cpu_addr, dma_handle, &attrs);
  496. +}
  497. +
  498. +
  499. #endif /* _ASM_DMA_MAPPING_H */
  500. --- a/arch/mips/mm/dma-default.c
  501. +++ b/arch/mips/mm/dma-default.c
  502. @@ -46,35 +46,6 @@ static int __init setnocoherentio(char *
  503. early_param("nocoherentio", setnocoherentio);
  504. #endif
  505. -static inline struct page *dma_addr_to_page(struct device *dev,
  506. - dma_addr_t dma_addr)
  507. -{
  508. - return pfn_to_page(
  509. - plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
  510. -}
  511. -
  512. -/*
  513. - * The affected CPUs below in 'cpu_needs_post_dma_flush()' can
  514. - * speculatively fill random cachelines with stale data at any time,
  515. - * requiring an extra flush post-DMA.
  516. - *
  517. - * Warning on the terminology - Linux calls an uncached area coherent;
  518. - * MIPS terminology calls memory areas with hardware maintained coherency
  519. - * coherent.
  520. - *
  521. - * Note that the R14000 and R16000 should also be checked for in this
  522. - * condition. However this function is only called on non-I/O-coherent
  523. - * systems and only the R10000 and R12000 are used in such systems, the
  524. - * SGI IP28 Indigo² rsp. SGI IP32 aka O2.
  525. - */
  526. -static inline int cpu_needs_post_dma_flush(struct device *dev)
  527. -{
  528. - return !plat_device_is_coherent(dev) &&
  529. - (boot_cpu_type() == CPU_R10000 ||
  530. - boot_cpu_type() == CPU_R12000 ||
  531. - boot_cpu_type() == CPU_BMIPS5000);
  532. -}
  533. -
  534. static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
  535. {
  536. gfp_t dma_flag;
  537. @@ -129,7 +100,7 @@ static void *mips_dma_alloc_noncoherent(
  538. return ret;
  539. }
  540. -static void *mips_dma_alloc_coherent(struct device *dev, size_t size,
  541. +void *mips_dma_alloc_coherent(struct device *dev, size_t size,
  542. dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs)
  543. {
  544. void *ret;
  545. @@ -165,6 +136,7 @@ static void *mips_dma_alloc_coherent(str
  546. return ret;
  547. }
  548. +EXPORT_SYMBOL(mips_dma_alloc_coherent);
  549. static void mips_dma_free_noncoherent(struct device *dev, size_t size,
  550. @@ -174,7 +146,7 @@ static void mips_dma_free_noncoherent(st
  551. free_pages((unsigned long) vaddr, get_order(size));
  552. }
  553. -static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
  554. +void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
  555. dma_addr_t dma_handle, struct dma_attrs *attrs)
  556. {
  557. unsigned long addr = (unsigned long) vaddr;
  558. @@ -196,40 +168,7 @@ static void mips_dma_free_coherent(struc
  559. if (!dma_release_from_contiguous(dev, page, count))
  560. __free_pages(page, get_order(size));
  561. }
  562. -
  563. -static int mips_dma_mmap(struct device *dev, struct vm_area_struct *vma,
  564. - void *cpu_addr, dma_addr_t dma_addr, size_t size,
  565. - struct dma_attrs *attrs)
  566. -{
  567. - unsigned long user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
  568. - unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
  569. - unsigned long addr = (unsigned long)cpu_addr;
  570. - unsigned long off = vma->vm_pgoff;
  571. - unsigned long pfn;
  572. - int ret = -ENXIO;
  573. -
  574. - if (!plat_device_is_coherent(dev) && !hw_coherentio)
  575. - addr = CAC_ADDR(addr);
  576. -
  577. - pfn = page_to_pfn(virt_to_page((void *)addr));
  578. -
  579. - if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs))
  580. - vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
  581. - else
  582. - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
  583. -
  584. - if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
  585. - return ret;
  586. -
  587. - if (off < count && user_count <= (count - off)) {
  588. - ret = remap_pfn_range(vma, vma->vm_start,
  589. - pfn + off,
  590. - user_count << PAGE_SHIFT,
  591. - vma->vm_page_prot);
  592. - }
  593. -
  594. - return ret;
  595. -}
  596. +EXPORT_SYMBOL(mips_dma_free_coherent);
  597. static inline void __dma_sync_virtual(void *addr, size_t size,
  598. enum dma_data_direction direction)
  599. @@ -258,7 +197,7 @@ static inline void __dma_sync_virtual(vo
  600. * If highmem is not configured then the bulk of this loop gets
  601. * optimized out.
  602. */
  603. -static inline void __dma_sync(struct page *page,
  604. +void __dma_sync(struct page *page,
  605. unsigned long offset, size_t size, enum dma_data_direction direction)
  606. {
  607. size_t left = size;
  608. @@ -288,120 +227,7 @@ static inline void __dma_sync(struct pag
  609. left -= len;
  610. } while (left);
  611. }
  612. -
  613. -static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
  614. - size_t size, enum dma_data_direction direction, struct dma_attrs *attrs)
  615. -{
  616. - if (cpu_needs_post_dma_flush(dev))
  617. - __dma_sync(dma_addr_to_page(dev, dma_addr),
  618. - dma_addr & ~PAGE_MASK, size, direction);
  619. - plat_post_dma_flush(dev);
  620. - plat_unmap_dma_mem(dev, dma_addr, size, direction);
  621. -}
  622. -
  623. -static int mips_dma_map_sg(struct device *dev, struct scatterlist *sglist,
  624. - int nents, enum dma_data_direction direction, struct dma_attrs *attrs)
  625. -{
  626. - int i;
  627. - struct scatterlist *sg;
  628. -
  629. - for_each_sg(sglist, sg, nents, i) {
  630. - if (!plat_device_is_coherent(dev))
  631. - __dma_sync(sg_page(sg), sg->offset, sg->length,
  632. - direction);
  633. -#ifdef CONFIG_NEED_SG_DMA_LENGTH
  634. - sg->dma_length = sg->length;
  635. -#endif
  636. - sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
  637. - sg->offset;
  638. - }
  639. -
  640. - return nents;
  641. -}
  642. -
  643. -static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page,
  644. - unsigned long offset, size_t size, enum dma_data_direction direction,
  645. - struct dma_attrs *attrs)
  646. -{
  647. - if (!plat_device_is_coherent(dev))
  648. - __dma_sync(page, offset, size, direction);
  649. -
  650. - return plat_map_dma_mem_page(dev, page) + offset;
  651. -}
  652. -
  653. -static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
  654. - int nhwentries, enum dma_data_direction direction,
  655. - struct dma_attrs *attrs)
  656. -{
  657. - int i;
  658. - struct scatterlist *sg;
  659. -
  660. - for_each_sg(sglist, sg, nhwentries, i) {
  661. - if (!plat_device_is_coherent(dev) &&
  662. - direction != DMA_TO_DEVICE)
  663. - __dma_sync(sg_page(sg), sg->offset, sg->length,
  664. - direction);
  665. - plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
  666. - }
  667. -}
  668. -
  669. -static void mips_dma_sync_single_for_cpu(struct device *dev,
  670. - dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
  671. -{
  672. - if (cpu_needs_post_dma_flush(dev))
  673. - __dma_sync(dma_addr_to_page(dev, dma_handle),
  674. - dma_handle & ~PAGE_MASK, size, direction);
  675. - plat_post_dma_flush(dev);
  676. -}
  677. -
  678. -static void mips_dma_sync_single_for_device(struct device *dev,
  679. - dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
  680. -{
  681. - if (!plat_device_is_coherent(dev))
  682. - __dma_sync(dma_addr_to_page(dev, dma_handle),
  683. - dma_handle & ~PAGE_MASK, size, direction);
  684. -}
  685. -
  686. -static void mips_dma_sync_sg_for_cpu(struct device *dev,
  687. - struct scatterlist *sglist, int nelems,
  688. - enum dma_data_direction direction)
  689. -{
  690. - int i;
  691. - struct scatterlist *sg;
  692. -
  693. - if (cpu_needs_post_dma_flush(dev)) {
  694. - for_each_sg(sglist, sg, nelems, i) {
  695. - __dma_sync(sg_page(sg), sg->offset, sg->length,
  696. - direction);
  697. - }
  698. - }
  699. - plat_post_dma_flush(dev);
  700. -}
  701. -
  702. -static void mips_dma_sync_sg_for_device(struct device *dev,
  703. - struct scatterlist *sglist, int nelems,
  704. - enum dma_data_direction direction)
  705. -{
  706. - int i;
  707. - struct scatterlist *sg;
  708. -
  709. - if (!plat_device_is_coherent(dev)) {
  710. - for_each_sg(sglist, sg, nelems, i) {
  711. - __dma_sync(sg_page(sg), sg->offset, sg->length,
  712. - direction);
  713. - }
  714. - }
  715. -}
  716. -
  717. -int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
  718. -{
  719. - return 0;
  720. -}
  721. -
  722. -int mips_dma_supported(struct device *dev, u64 mask)
  723. -{
  724. - return plat_dma_supported(dev, mask);
  725. -}
  726. +EXPORT_SYMBOL(__dma_sync);
  727. void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
  728. enum dma_data_direction direction)
  729. @@ -414,24 +240,10 @@ void dma_cache_sync(struct device *dev,
  730. EXPORT_SYMBOL(dma_cache_sync);
  731. -static struct dma_map_ops mips_default_dma_map_ops = {
  732. - .alloc = mips_dma_alloc_coherent,
  733. - .free = mips_dma_free_coherent,
  734. - .mmap = mips_dma_mmap,
  735. - .map_page = mips_dma_map_page,
  736. - .unmap_page = mips_dma_unmap_page,
  737. - .map_sg = mips_dma_map_sg,
  738. - .unmap_sg = mips_dma_unmap_sg,
  739. - .sync_single_for_cpu = mips_dma_sync_single_for_cpu,
  740. - .sync_single_for_device = mips_dma_sync_single_for_device,
  741. - .sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
  742. - .sync_sg_for_device = mips_dma_sync_sg_for_device,
  743. - .mapping_error = mips_dma_mapping_error,
  744. - .dma_supported = mips_dma_supported
  745. -};
  746. -
  747. -struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
  748. +#ifdef CONFIG_SYS_HAS_DMA_OPS
  749. +struct dma_map_ops *mips_dma_map_ops = NULL;
  750. EXPORT_SYMBOL(mips_dma_map_ops);
  751. +#endif
  752. #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)