Merge branch 'CR_5620_DMA_NONCOHERENT_samin.guo' into 'jh7110-5.15.y-devel'

CR5620： Add non-coherent DMA handling

See merge request sdk/linux!881

6 files changed, 152 insertions, 0 deletions

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 6fe5d24bcf30..2a48a038cbd9 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -220,6 +220,17 @@ config PGTABLE_LEVELS
 config LOCKDEP_SUPPORT
 	def_bool y
 
+config RISCV_DMA_NONCOHERENT
+	bool "Support non-coherent DMA"
+	default SOC_STARFIVE
+	select ARCH_HAS_DMA_PREP_COHERENT
+	select ARCH_HAS_DMA_SET_UNCACHED
+	select ARCH_HAS_DMA_CLEAR_UNCACHED
+	select ARCH_HAS_SYNC_DMA_FOR_DEVICE
+	select ARCH_HAS_SYNC_DMA_FOR_CPU
+	select ARCH_HAS_SETUP_DMA_OPS
+	select DMA_DIRECT_REMAP
+
 source "arch/riscv/Kconfig.socs"
 source "arch/riscv/Kconfig.erratas"
 
diff --git a/arch/riscv/boot/dts/starfive/jh7110.dtsi b/arch/riscv/boot/dts/starfive/jh7110.dtsi
index 3541fb432334..4baa10480b95 100755
--- a/arch/riscv/boot/dts/starfive/jh7110.dtsi
+++ b/arch/riscv/boot/dts/starfive/jh7110.dtsi
@@ -246,6 +246,7 @@
 			cache-sets = <2048>;
 			cache-size = <2097152>;
 			cache-unified;
+			uncached-offset = <0x4 0x00000000>;
 		};
 
 		aon_syscon: aon_syscon@17010000 {
@@ -460,6 +461,7 @@
 			ranges;
 			usbdrd_cdns3: usb@10100000 {
 				compatible = "cdns,usb3";
+				dma-coherent;
 				reg = <0x0 0x10100000 0x0 0x10000>,
 				      <0x0 0x10110000 0x0 0x10000>,
 				      <0x0 0x10120000 0x0 0x10000>;
@@ -620,6 +622,7 @@
 
 		dma: dma-controller@16050000 {
 			compatible = "starfive,jh7110-dma", "snps,axi-dma-1.01a";
+			dma-coherent;
 			reg = <0x0 0x16050000 0x0 0x10000>;
 			clocks = <&clkgen JH7110_DMA1P_CLK_AXI>,
 				 <&clkgen JH7110_DMA1P_CLK_AHB>,
@@ -729,6 +732,7 @@
 
 		sec_dma: sec_dma@16008000 {
 			compatible = "arm,pl080", "arm,primecell";
+			dma-coherent;
 			arm,primecell-periphid = <0x00041080>;
 			reg = <0x0 0x16008000 0x0 0x4000>;
 			reg-names = "sec_dma";
@@ -748,6 +752,7 @@
 
 		crypto: crypto@16000000 {
 			compatible = "starfive,jh7110-sec";
+			dma-coherent;
 			reg = <0x0 0x16000000 0x0 0x4000>,
 			      <0x0 0x16008000 0x0 0x4000>;
 			reg-names = "secreg","secdma";
@@ -860,6 +865,7 @@
 		/* unremovable emmc as mmcblk0 */
 		sdio0: sdio0@16010000 {
 			compatible = "starfive,jh7110-sdio";
+			dma-coherent;
 			reg = <0x0 0x16010000 0x0 0x10000>;
 			clocks = <&clkgen JH7110_SDIO0_CLK_AHB>,
 				 <&clkgen JH7110_SDIO0_CLK_SDCARD>;
@@ -876,6 +882,7 @@
 
 		sdio1: sdio1@16020000 {
 			compatible = "starfive,jh7110-sdio";
+			dma-coherent;
 			reg = <0x0 0x16020000 0x0 0x10000>;
 			clocks = <&clkgen JH7110_SDIO1_CLK_AHB>,
 				 <&clkgen JH7110_SDIO1_CLK_SDCARD>;
@@ -1035,6 +1042,7 @@
 
 		gmac0: ethernet@16030000 {
 			compatible = "starfive,dwmac","snps,dwmac-5.10a";
+			dma-coherent;
 			reg = <0x0 0x16030000 0x0 0x10000>;
 			clock-names = "gtx",
 				"tx",
@@ -1076,6 +1084,7 @@
 
 		gmac1: ethernet@16040000 {
 			compatible = "starfive,dwmac","snps,dwmac-5.10a";
+			dma-coherent;
 			reg = <0x0 0x16040000 0x0 0x10000>;
 			clock-names = "gtx",
 				"tx",
@@ -1522,6 +1531,7 @@
 
 		pcie0: pcie@2B000000 {
 			compatible = "starfive,jh7110-pcie","plda,pci-xpressrich3-axi";
+			dma-coherent;
 			#address-cells = <3>;
 			#size-cells = <2>;
 			#interrupt-cells = <1>;
@@ -1562,6 +1572,7 @@
 
 		pcie1: pcie@2C000000 {
 			compatible = "starfive,jh7110-pcie","plda,pci-xpressrich3-axi";
+			dma-coherent;
 			#address-cells = <3>;
 			#size-cells = <2>;
 			#interrupt-cells = <1>;
@@ -1806,6 +1817,7 @@
 
 		co_process: e24@0 {
 			compatible = "starfive,e24";
+			dma-coherent;
 			reg = <0x0 0xc0110000 0x0 0x00001000>,
 				<0x0 0xc0111000 0x0 0x0001f000>;
 			reg-names = "ecmd", "espace";
@@ -1830,6 +1842,7 @@
 
 		xrp: xrp@0 {
 			compatible = "cdns,xrp";
+			dma-coherent;
 			reg = <0x0  0x10230000 0x0 0x00010000
 				0x0  0x10240000 0x0 0x00010000>;
 			memory-region = <&xrp_reserved>;
diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile
index 7ebaef10ea1b..959bef49098b 100644
--- a/arch/riscv/mm/Makefile
+++ b/arch/riscv/mm/Makefile
@@ -27,3 +27,4 @@ KASAN_SANITIZE_init.o := n
 endif
 
 obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o
+obj-$(CONFIG_RISCV_DMA_NONCOHERENT) += dma-noncoherent.o
diff --git a/arch/riscv/mm/dma-noncoherent.c b/arch/riscv/mm/dma-noncoherent.c
new file mode 100644
index 000000000000..86d23046b958
--- /dev/null
+++ b/arch/riscv/mm/dma-noncoherent.c
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * RISC-V specific functions to support DMA for non-coherent devices
+ *
+ * Copyright (c) 2021 Western Digital Corporation or its affiliates.
+ */
+
+#include <linux/dma-map-ops.h>
+
+#include <soc/sifive/sifive_l2_cache.h>
+
+void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, enum dma_data_direction dir)
+{
+	if (sifive_l2_handle_noncoherent())
+		sifive_l2_flush_range(paddr, size);
+}
+
+void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, enum dma_data_direction dir)
+{
+	if (sifive_l2_handle_noncoherent())
+		sifive_l2_flush_range(paddr, size);
+}
+
+void *arch_dma_set_uncached(void *addr, size_t size)
+{
+	if (sifive_l2_handle_noncoherent())
+		return sifive_l2_set_uncached(addr, size);
+
+	return addr;
+}
+
+void arch_dma_clear_uncached(void *addr, size_t size)
+{
+	if (sifive_l2_handle_noncoherent())
+		sifive_l2_clear_uncached(addr, size);
+}
+
+void arch_dma_prep_coherent(struct page *page, size_t size)
+{
+	void *flush_addr = page_address(page);
+
+	memset(flush_addr, 0, size);
+	if (sifive_l2_handle_noncoherent())
+		sifive_l2_flush_range(__pa(flush_addr), size);
+}
+
+void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
+		const struct iommu_ops *iommu, bool coherent)
+{
+	/* If a specific device is dma-coherent, set it here */
+	dev->dma_coherent = coherent;
+}
diff --git a/drivers/soc/sifive/sifive_l2_cache.c b/drivers/soc/sifive/sifive_l2_cache.c
index bf824d52b9b8..ce29ae99dd24 100644
--- a/drivers/soc/sifive/sifive_l2_cache.c
+++ b/drivers/soc/sifive/sifive_l2_cache.c
@@ -5,12 +5,14 @@
  * Copyright (C) 2018-2019 SiFive, Inc.
  *
  */
+#include <linux/align.h>
 #include <linux/debugfs.h>
 #include <linux/interrupt.h>
 #include <linux/of_irq.h>
 #include <linux/of_address.h>
 #include <linux/device.h>
 #include <asm/cacheinfo.h>
+#include <asm/page.h>
 #include <soc/sifive/sifive_l2_cache.h>
 
 #define SIFIVE_L2_DIRECCFIX_LOW 0x100
@@ -30,6 +32,7 @@
 #define SIFIVE_L2_DATECCFAIL_COUNT 0x168
 
 #define SIFIVE_L2_FLUSH64 0x200
+#define SIFIVE_L2_FLUSH32 0x240
 
 #define SIFIVE_L2_CONFIG 0x00
 #define SIFIVE_L2_WAYENABLE 0x08
@@ -41,6 +44,8 @@
 static void __iomem *l2_base;
 static int g_irq[SIFIVE_L2_MAX_ECCINTR];
 static struct riscv_cacheinfo_ops l2_cache_ops;
+static phys_addr_t uncached_offset;
+DEFINE_STATIC_KEY_FALSE(sifive_l2_handle_noncoherent_key);
 
 enum {
 	DIR_CORR = 0,
@@ -119,6 +124,48 @@ int unregister_sifive_l2_error_notifier(struct notifier_block *nb)
 }
 EXPORT_SYMBOL_GPL(unregister_sifive_l2_error_notifier);
 
+void sifive_l2_flush_range(phys_addr_t start, size_t len)
+{
+	phys_addr_t end = start + len;
+	phys_addr_t line;
+
+	if (!len)
+		return;
+
+	mb();
+	for (line = ALIGN_DOWN(start, SIFIVE_L2_FLUSH64_LINE_LEN); line < end;
+	     line += SIFIVE_L2_FLUSH64_LINE_LEN) {
+#ifdef CONFIG_32BIT
+		writel(line >> 4, l2_base + SIFIVE_L2_FLUSH32);
+#else
+		writeq(line, l2_base + SIFIVE_L2_FLUSH64);
+#endif
+		mb();
+	}
+}
+EXPORT_SYMBOL_GPL(sifive_l2_flush_range);
+
+void sifive_ccache_flush_range(phys_addr_t start, size_t len)
+{
+	sifive_l2_flush_range(start, len);
+}
+EXPORT_SYMBOL_GPL(sifive_ccache_flush_range);
+
+void *sifive_l2_set_uncached(void *addr, size_t size)
+{
+	phys_addr_t phys_addr = __pa(addr) + uncached_offset;
+	void *mem_base;
+
+	mem_base = memremap(phys_addr, size, MEMREMAP_WT);
+	if (!mem_base) {
+		pr_err("%s memremap failed for addr %p\n", __func__, addr);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return mem_base;
+}
+EXPORT_SYMBOL_GPL(sifive_l2_set_uncached);
+
 #ifdef CONFIG_SIFIVE_L2_FLUSH
 void sifive_l2_flush64_range(unsigned long start, unsigned long len)
 {
@@ -235,6 +282,7 @@ static int __init sifive_l2_init(void)
 	struct device_node *np;
 	struct resource res;
 	int i, rc, intr_num;
+	u64 offset;
 
 	np = of_find_matching_node(NULL, sifive_l2_ids);
 	if (!np)
@@ -262,6 +310,11 @@ static int __init sifive_l2_init(void)
 		}
 	}
 
+	if (!of_property_read_u64(np, "uncached-offset", &offset)) {
+		uncached_offset = offset;
+		static_branch_enable(&sifive_l2_handle_noncoherent_key);
+	}
+
 	l2_config_read();
 
 	l2_cache_ops.get_priv_group = l2_get_priv_group;
diff --git a/include/soc/sifive/sifive_l2_cache.h b/include/soc/sifive/sifive_l2_cache.h
index 04dcac10b5cd..f472e4cb1232 100644
--- a/include/soc/sifive/sifive_l2_cache.h
+++ b/include/soc/sifive/sifive_l2_cache.h
@@ -7,12 +7,34 @@
 #ifndef __SOC_SIFIVE_L2_CACHE_H
 #define __SOC_SIFIVE_L2_CACHE_H
 
+#include <linux/io.h>
+#include <linux/jump_label.h>
+
 extern int register_sifive_l2_error_notifier(struct notifier_block *nb);
 extern int unregister_sifive_l2_error_notifier(struct notifier_block *nb);
 
 #define SIFIVE_L2_ERR_TYPE_CE 0
 #define SIFIVE_L2_ERR_TYPE_UE 1
 
+DECLARE_STATIC_KEY_FALSE(sifive_l2_handle_noncoherent_key);
+
+static inline bool sifive_l2_handle_noncoherent(void)
+{
+#ifdef CONFIG_SIFIVE_L2
+	return static_branch_unlikely(&sifive_l2_handle_noncoherent_key);
+#else
+	return false;
+#endif
+}
+
+void sifive_ccache_flush_range(phys_addr_t start, size_t len);
+void sifive_l2_flush_range(phys_addr_t start, size_t len);
+void *sifive_l2_set_uncached(void *addr, size_t size);
+static inline void sifive_l2_clear_uncached(void *addr, size_t size)
+{
+	memunmap(addr);
+}
+
 #ifdef CONFIG_SIFIVE_L2_FLUSH
 void sifive_l2_flush64_range(unsigned long start, unsigned long len);
 #endif